You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2017/11/19 00:03:16 UTC

[JENKINS] Lucene-Solr-Tests-7.x - Build # 246 - Still Unstable

Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/246/

1 tests failed.
FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitStaticIndexReplication

Error Message:
Could not load collection from ZK: control_collection

Stack Trace:
org.apache.solr.common.SolrException: Could not load collection from ZK: control_collection
	at __randomizedtesting.SeedInfo.seed([93B52C3220965026:D9FFB832B13E7F23]:0)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1122)
	at org.apache.solr.common.cloud.ZkStateReader$LazyCollectionRef.get(ZkStateReader.java:647)
	at org.apache.solr.common.cloud.ClusterState.getCollectionOrNull(ClusterState.java:130)
	at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:110)
	at org.apache.solr.cloud.ShardSplitTest.testSplitStaticIndexReplication(ShardSplitTest.java:189)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/control_collection/state.json
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
	at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:332)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:332)
	at org.apache.solr.common.cloud.ZkStateReader.fetchCollectionState(ZkStateReader.java:1134)
	at org.apache.solr.common.cloud.ZkStateReader.getCollectionLive(ZkStateReader.java:1120)
	... 45 more




Build Log:
[...truncated 11926 lines...]
   [junit4] Suite: org.apache.solr.cloud.ShardSplitTest
   [junit4]   2> 169170 INFO  (SUITE-ShardSplitTest-seed#[93B52C3220965026]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/init-core-data-001
   [junit4]   2> 169185 WARN  (SUITE-ShardSplitTest-seed#[93B52C3220965026]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
   [junit4]   2> 169185 INFO  (SUITE-ShardSplitTest-seed#[93B52C3220965026]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
   [junit4]   2> 169187 INFO  (SUITE-ShardSplitTest-seed#[93B52C3220965026]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 169187 INFO  (SUITE-ShardSplitTest-seed#[93B52C3220965026]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 169206 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 169222 INFO  (Thread-103) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 169222 INFO  (Thread-103) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 169279 ERROR (Thread-103) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 169331 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkTestServer start zk server on port:40119
   [junit4]   2> 169481 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 169535 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 169536 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 169537 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 169551 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 169552 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 169553 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 169554 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 169588 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 169589 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 169590 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractZkTestCase put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 169624 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fd0f3f9370001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 169644 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractFullDistribZkTestBase Will use TLOG replicas unless explicitly asked otherwise
   [junit4]   2> 170611 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 170612 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5d581ad0{/,null,AVAILABLE}
   [junit4]   2> 170614 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@79842a8{HTTP/1.1,[http/1.1]}{127.0.0.1:35745}
   [junit4]   2> 170614 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server Started @185831ms
   [junit4]   2> 170614 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=35745, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/control-001/cores}
   [junit4]   2> 170614 ERROR (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 170614 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 170614 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 170614 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 170614 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T21:06:12.262Z
   [junit4]   2> 170961 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 170961 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/control-001/solr.xml
   [junit4]   2> 171030 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 171030 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 171031 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 171103 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40119/solr
   [junit4]   2> 171819 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 171819 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35745_
   [junit4]   2> 171822 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.c.Overseer Overseer (id=99027463102201860-127.0.0.1:35745_-n_0000000000) starting
   [junit4]   2> 171929 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35745_
   [junit4]   2> 171965 INFO  (zkCallback-88-thread-1-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 172421 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 172479 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 172479 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 172480 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:35745_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/control-001/cores
   [junit4]   2> 172860 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 172860 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40119/solr ready
   [junit4]   2> 172869 INFO  (qtp75213006-494) [n:127.0.0.1:35745_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35745_&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 172898 INFO  (OverseerThreadFactory-286-thread-1-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.CreateCollectionCmd Create collection control_collection
   [junit4]   2> 173033 INFO  (qtp75213006-492) [n:127.0.0.1:35745_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 173033 INFO  (qtp75213006-492) [n:127.0.0.1:35745_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 173188 INFO  (zkCallback-88-thread-1-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 174118 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 174172 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
   [junit4]   2> 174571 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 174826 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
   [junit4]   2> 174826 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 174862 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 174863 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/control-001/cores/control_collection_shard1_replica_n1/data/]
   [junit4]   2> 174996 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=21.9443359375, floorSegmentMB=0.2509765625, forceMergeDeletesPctAllowed=16.887172007506212, segmentsPerTier=19.0, maxCFSSegmentSizeMB=0.9794921875, noCFSRatio=0.0
   [junit4]   2> 175053 WARN  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 175345 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 175345 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 175346 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 175346 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 175347 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8066031529201612]
   [junit4]   2> 175360 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@48d5d5f9[control_collection_shard1_replica_n1] main]
   [junit4]   2> 175361 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 175362 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 175362 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 175362 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584439416072437760
   [junit4]   2> 175382 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 175382 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 175382 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35745/control_collection_shard1_replica_n1/
   [junit4]   2> 175382 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 175382 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:35745/control_collection_shard1_replica_n1/ has no replicas
   [junit4]   2> 175382 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 175417 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35745/control_collection_shard1_replica_n1/ shard1
   [junit4]   2> 175419 INFO  (searcherExecutor-289-thread-1-processing-n:127.0.0.1:35745_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@48d5d5f9[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 175454 INFO  (zkCallback-88-thread-1-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 175519 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 175520 INFO  (qtp75213006-492) [n:127.0.0.1:35745_ c:control_collection s:shard1  x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2487
   [junit4]   2> 175860 INFO  (zkCallback-88-thread-1-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 175878 INFO  (qtp75213006-494) [n:127.0.0.1:35745_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 175878 INFO  (qtp75213006-494) [n:127.0.0.1:35745_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:35745_&wt=javabin&version=2} status=0 QTime=3009
   [junit4]   2> 176276 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 176276 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40119/solr ready
   [junit4]   2> 176276 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 176280 INFO  (qtp75213006-497) [n:127.0.0.1:35745_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 176405 INFO  (OverseerThreadFactory-286-thread-2-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 176406 WARN  (OverseerThreadFactory-286-thread-2-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
   [junit4]   2> 176430 INFO  (OverseerCollectionConfigSetProcessor-99027463102201860-127.0.0.1:35745_-n_0000000000) [n:127.0.0.1:35745_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 176776 INFO  (qtp75213006-497) [n:127.0.0.1:35745_    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 176776 INFO  (qtp75213006-497) [n:127.0.0.1:35745_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=496
   [junit4]   2> 177209 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-1-001 of type TLOG
   [junit4]   2> 177227 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 177251 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4d976b51{/,null,AVAILABLE}
   [junit4]   2> 177252 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@43ac17bd{HTTP/1.1,[http/1.1]}{127.0.0.1:42082}
   [junit4]   2> 177252 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server Started @192469ms
   [junit4]   2> 177252 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/tempDir-001/jetty1, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=42082, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-1-001/cores}
   [junit4]   2> 177252 ERROR (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 177252 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 177252 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 177252 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 177252 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T21:06:18.900Z
   [junit4]   2> 177325 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 177325 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-1-001/solr.xml
   [junit4]   2> 177398 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 177398 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 177443 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 177499 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40119/solr
   [junit4]   2> 177747 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:42082_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 177749 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:42082_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 177784 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:42082_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42082_
   [junit4]   2> 177785 INFO  (zkCallback-96-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 177824 INFO  (zkCallback-101-thread-1-processing-n:127.0.0.1:42082_) [n:127.0.0.1:42082_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 177905 INFO  (zkCallback-88-thread-4-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 178295 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:42082_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 178368 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:42082_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 178368 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:42082_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 178382 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:42082_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-1-001/cores
   [junit4]   2> 178549 INFO  (qtp75213006-498) [n:127.0.0.1:35745_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:42082_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 178600 INFO  (OverseerCollectionConfigSetProcessor-99027463102201860-127.0.0.1:35745_-n_0000000000) [n:127.0.0.1:35745_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 178697 INFO  (OverseerThreadFactory-286-thread-3-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:42082_ for creating new replica
   [junit4]   2> 178781 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 178781 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 178927 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 179010 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema [collection1_shard2_replica_t41] Schema name=test
   [junit4]   2> 179742 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 179919 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t41' using configuration from collection collection1, trusted=true
   [junit4]   2> 179919 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t41' (registry 'solr.core.collection1.shard2.replica_t41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 179919 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 179919 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [[collection1_shard2_replica_t41] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-1-001/cores/collection1_shard2_replica_t41], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-1-001/cores/collection1_shard2_replica_t41/data/]
   [junit4]   2> 179955 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=21.9443359375, floorSegmentMB=0.2509765625, forceMergeDeletesPctAllowed=16.887172007506212, segmentsPerTier=19.0, maxCFSSegmentSizeMB=0.9794921875, noCFSRatio=0.0
   [junit4]   2> 180067 WARN  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 180284 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 180284 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 180285 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 180285 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 180286 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8066031529201612]
   [junit4]   2> 180307 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.SolrIndexSearcher Opening [Searcher@3c4c8f78[collection1_shard2_replica_t41] main]
   [junit4]   2> 180341 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 180341 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 180342 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 180342 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584439421294346240
   [junit4]   2> 180435 INFO  (searcherExecutor-300-thread-1-processing-n:127.0.0.1:42082_ x:collection1_shard2_replica_t41 s:shard2 c:collection1) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SolrCore [collection1_shard2_replica_t41] Registered new searcher Searcher@3c4c8f78[collection1_shard2_replica_t41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 180451 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 180451 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 180452 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:42082/collection1_shard2_replica_t41/
   [junit4]   2> 180452 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 180452 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.SyncStrategy http://127.0.0.1:42082/collection1_shard2_replica_t41/ has no replicas
   [junit4]   2> 180452 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 180452 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController collection1_shard2_replica_t41 stopping background replication from leader
   [junit4]   2> 180454 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:42082/collection1_shard2_replica_t41/ shard2
   [junit4]   2> 180629 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 180646 INFO  (qtp1092800812-543) [n:127.0.0.1:42082_ c:collection1 s:shard2  x:collection1_shard2_replica_t41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=1865
   [junit4]   2> 180684 INFO  (qtp75213006-498) [n:127.0.0.1:35745_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:42082_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2134
   [junit4]   2> 180684 INFO  (OverseerCollectionConfigSetProcessor-99027463102201860-127.0.0.1:35745_-n_0000000000) [n:127.0.0.1:35745_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 181380 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-2-001 of type TLOG
   [junit4]   2> 181402 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 181404 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@383b26b9{/,null,AVAILABLE}
   [junit4]   2> 181404 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@28bba5ac{HTTP/1.1,[http/1.1]}{127.0.0.1:46634}
   [junit4]   2> 181404 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server Started @196621ms
   [junit4]   2> 181404 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/tempDir-001/jetty2, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=46634, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-2-001/cores}
   [junit4]   2> 181404 ERROR (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 181405 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 181405 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 181405 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 181405 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T21:06:23.053Z
   [junit4]   2> 181556 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 181556 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-2-001/solr.xml
   [junit4]   2> 181559 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 181559 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 181593 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 181633 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40119/solr
   [junit4]   2> 181865 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:46634_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 181867 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:46634_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 181881 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:46634_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46634_
   [junit4]   2> 181902 INFO  (zkCallback-96-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 181902 INFO  (zkCallback-101-thread-1-processing-n:127.0.0.1:42082_) [n:127.0.0.1:42082_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 181979 INFO  (zkCallback-88-thread-4-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 181962 INFO  (zkCallback-107-thread-1-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
   [junit4]   2> 182370 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:46634_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 182459 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:46634_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 182459 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:46634_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 182461 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:46634_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-2-001/cores
   [junit4]   2> 182717 INFO  (qtp75213006-491) [n:127.0.0.1:35745_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:46634_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 182932 INFO  (OverseerThreadFactory-286-thread-4-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:46634_ for creating new replica
   [junit4]   2> 182991 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 182992 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 184196 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 184453 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema [collection1_shard1_replica_t43] Schema name=test
   [junit4]   2> 185361 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 185489 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_t43' using configuration from collection collection1, trusted=true
   [junit4]   2> 185540 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_t43' (registry 'solr.core.collection1.shard1.replica_t43') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 185540 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 185541 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [[collection1_shard1_replica_t43] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-2-001/cores/collection1_shard1_replica_t43], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-2-001/cores/collection1_shard1_replica_t43/data/]
   [junit4]   2> 185613 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=21.9443359375, floorSegmentMB=0.2509765625, forceMergeDeletesPctAllowed=16.887172007506212, segmentsPerTier=19.0, maxCFSSegmentSizeMB=0.9794921875, noCFSRatio=0.0
   [junit4]   2> 185731 WARN  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 186241 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 186241 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 186242 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 186242 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 186257 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8066031529201612]
   [junit4]   2> 186258 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.SolrIndexSearcher Opening [Searcher@dac8f9c[collection1_shard1_replica_t43] main]
   [junit4]   2> 186314 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 186314 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 186315 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 186315 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584439427557490688
   [junit4]   2> 186343 INFO  (searcherExecutor-311-thread-1-processing-n:127.0.0.1:46634_ x:collection1_shard1_replica_t43 s:shard1 c:collection1) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SolrCore [collection1_shard1_replica_t43] Registered new searcher Searcher@dac8f9c[collection1_shard1_replica_t43] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 186353 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 186353 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 186353 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46634/collection1_shard1_replica_t43/
   [junit4]   2> 186353 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 186353 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.SyncStrategy http://127.0.0.1:46634/collection1_shard1_replica_t43/ has no replicas
   [junit4]   2> 186353 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 186353 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController collection1_shard1_replica_t43 stopping background replication from leader
   [junit4]   2> 186388 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46634/collection1_shard1_replica_t43/ shard1
   [junit4]   2> 186440 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 186463 INFO  (qtp1251775273-572) [n:127.0.0.1:46634_ c:collection1 s:shard1  x:collection1_shard1_replica_t43] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_t43&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=3472
   [junit4]   2> 186514 INFO  (qtp75213006-491) [n:127.0.0.1:35745_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:46634_&action=ADDREPLICA&collection=collection1&shard=shard1&type=TLOG&wt=javabin&version=2} status=0 QTime=3796
   [junit4]   2> 186761 INFO  (OverseerCollectionConfigSetProcessor-99027463102201860-127.0.0.1:35745_-n_0000000000) [n:127.0.0.1:35745_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 187226 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-3-001 of type TLOG
   [junit4]   2> 187226 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 187346 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66f5cd49{/,null,AVAILABLE}
   [junit4]   2> 187346 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@1d9c7b93{HTTP/1.1,[http/1.1]}{127.0.0.1:34723}
   [junit4]   2> 187346 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server Started @202563ms
   [junit4]   2> 187347 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/tempDir-001/jetty3, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=34723, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-3-001/cores}
   [junit4]   2> 187347 ERROR (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 187422 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 187422 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 187422 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 187422 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T21:06:29.070Z
   [junit4]   2> 187438 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 187438 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-3-001/solr.xml
   [junit4]   2> 187442 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 187442 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 187443 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 187446 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40119/solr
   [junit4]   2> 187632 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:34723_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
   [junit4]   2> 187633 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:34723_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 187667 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:34723_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34723_
   [junit4]   2> 187683 INFO  (zkCallback-107-thread-1-processing-n:127.0.0.1:46634_) [n:127.0.0.1:46634_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 187720 INFO  (zkCallback-88-thread-1-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 187728 INFO  (zkCallback-96-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 187728 INFO  (zkCallback-101-thread-1-processing-n:127.0.0.1:42082_) [n:127.0.0.1:42082_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 187781 INFO  (zkCallback-113-thread-1-processing-n:127.0.0.1:34723_) [n:127.0.0.1:34723_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
   [junit4]   2> 188207 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:34723_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 188267 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:34723_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 188267 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:34723_    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 188281 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:34723_    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-3-001/cores
   [junit4]   2> 188456 INFO  (qtp2019283656-601) [n:127.0.0.1:34723_    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:34723_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 188474 INFO  (OverseerThreadFactory-286-thread-5-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:34723_ for creating new replica
   [junit4]   2> 188493 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG
   [junit4]   2> 188509 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
   [junit4]   2> 189698 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 189856 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema [collection1_shard2_replica_t45] Schema name=test
   [junit4]   2> 190815 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
   [junit4]   2> 190946 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_t45' using configuration from collection collection1, trusted=true
   [junit4]   2> 190977 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_t45' (registry 'solr.core.collection1.shard2.replica_t45') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677
   [junit4]   2> 190977 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 190977 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [[collection1_shard2_replica_t45] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-3-001/cores/collection1_shard2_replica_t45], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-3-001/cores/collection1_shard2_replica_t45/data/]
   [junit4]   2> 190998 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=21.9443359375, floorSegmentMB=0.2509765625, forceMergeDeletesPctAllowed=16.887172007506212, segmentsPerTier=19.0, maxCFSSegmentSizeMB=0.9794921875, noCFSRatio=0.0
   [junit4]   2> 191070 WARN  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
   [junit4]   2> 191300 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 191300 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 191301 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 191301 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 191303 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=47, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8066031529201612]
   [junit4]   2> 191321 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d8bfc9[collection1_shard2_replica_t45] main]
   [junit4]   2> 191322 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 191322 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 191323 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 191324 INFO  (searcherExecutor-322-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@1d8bfc9[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 191324 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584439432809807872
   [junit4]   2> 191328 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.c.ZkController Core needs to recover:collection1_shard2_replica_t45
   [junit4]   2> 191362 INFO  (qtp2019283656-599) [n:127.0.0.1:34723_ c:collection1 s:shard2  x:collection1_shard2_replica_t45] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_t45&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=TLOG} status=0 QTime=2869
   [junit4]   2> 191362 INFO  (updateExecutor-110-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DefaultSolrCoreState Running recovery
   [junit4]   2> 191394 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
   [junit4]   2> 191394 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
   [junit4]   2> 191394 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.ZkController collection1_shard2_replica_t45 stopping background replication from leader
   [junit4]   2> 191394 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard2_replica_t45]
   [junit4]   2> 191394 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
   [junit4]   2> 191394 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard2_replica_t45] as recovering, leader is [http://127.0.0.1:42082/collection1_shard2_replica_t41/] and I am [http://127.0.0.1:34723/collection1_shard2_replica_t45/]
   [junit4]   2> 191427 INFO  (qtp2019283656-601) [n:127.0.0.1:34723_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:34723_&action=ADDREPLICA&collection=collection1&shard=shard2&type=TLOG&wt=javabin&version=2} status=0 QTime=2970
   [junit4]   2> 191451 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:42082]; [WaitForState: action=PREPRECOVERY&core=collection1_shard2_replica_t41&nodeName=127.0.0.1:34723_&coreNodeName=core_node46&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
   [junit4]   2> 191520 INFO  (qtp1092800812-541) [n:127.0.0.1:42082_    ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
   [junit4]   2> 191524 INFO  (qtp1092800812-541) [n:127.0.0.1:42082_    ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard2, thisCore=collection1_shard2_replica_t41, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:34723_, coreNodeName=core_node46, onlyIfActiveCheckResult=false, nodeProps: core_node46:{"core":"collection1_shard2_replica_t45","base_url":"http://127.0.0.1:34723","node_name":"127.0.0.1:34723_","state":"recovering","type":"TLOG"}
   [junit4]   2> 191524 INFO  (qtp1092800812-541) [n:127.0.0.1:42082_    ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node46, state: recovering, checkLive: true, onlyIfLeader: true for: 0 seconds.
   [junit4]   2> 191524 INFO  (qtp1092800812-541) [n:127.0.0.1:42082_    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:34723_&onlyIfLeaderActive=true&core=collection1_shard2_replica_t41&coreNodeName=core_node46&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=4
   [junit4]   2> 192017 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-4-001 of type TLOG
   [junit4]   2> 192018 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 192081 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
   [junit4]   2> 192081 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:42082/collection1_shard2_replica_t41/].
   [junit4]   2> 192158 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a12fe2d{/,null,AVAILABLE}
   [junit4]   2> 192159 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.AbstractConnector Started ServerConnector@33034a16{HTTP/1.1,[http/1.1]}{127.0.0.1:33626}
   [junit4]   2> 192159 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.e.j.s.Server Started @207376ms
   [junit4]   2> 192159 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/tempDir-001/jetty4, replicaType=TLOG, solrconfig=solrconfig.xml, hostContext=/, hostPort=33626, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-4-001/cores}
   [junit4]   2> 192159 ERROR (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 192163 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 192163 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 192163 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 192163 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-18T21:06:33.811Z
   [junit4]   2> 192202 INFO  (qtp1092800812-545) [n:127.0.0.1:42082_ c:collection1 s:shard2 r:core_node42 x:collection1_shard2_replica_t41] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1584439433730457600,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 192235 INFO  (qtp1092800812-545) [n:127.0.0.1:42082_ c:collection1 s:shard2 r:core_node42 x:collection1_shard2_replica_t41] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 192236 INFO  (qtp1092800812-545) [n:127.0.0.1:42082_ c:collection1 s:shard2 r:core_node42 x:collection1_shard2_replica_t41] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 192236 INFO  (qtp1092800812-545) [n:127.0.0.1:42082_ c:collection1 s:shard2 r:core_node42 x:collection1_shard2_replica_t41] o.a.s.u.p.LogUpdateProcessorFactory [collection1_shard2_replica_t41]  webapp= path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2}{commit=} 0 35
   [junit4]   2> 192341 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 192341 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001/shard-4-001/solr.xml
   [junit4]   2> 192345 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
   [junit4]   2> 192345 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
   [junit4]   2> 192437 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@303e1677, but no JMX reporters were configured - adding default JMX reporter.
   [junit4]   2> 192455 INFO  (qtp1092800812-547) [n:127.0.0.1:42082_ c:collection1 s:shard2 r:core_node42 x:collection1_shard2_replica_t41] o.a.s.c.S.Request [collection1_shard2_replica_t41]  webapp= path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 192456 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.h.IndexFetcher Master's generation: 1
   [junit4]   2> 192456 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.h.IndexFetcher Master's version: 0
   [junit4]   2> 192456 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.h.IndexFetcher Slave's generation: 1
   [junit4]   2> 192456 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.h.IndexFetcher Slave's version: 0
   [junit4]   2> 192456 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
   [junit4]   2> 192510 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40119/solr
   [junit4]   2> 192744 INFO  (OverseerCollectionConfigSetProcessor-99027463102201860-127.0.0.1:35745_-n_0000000000) [n:127.0.0.1:35745_    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 192745 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 192745 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@375186b6 commitCommandVersion:0
   [junit4]   2> 192928 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:33626_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
   [junit4]   2> 192930 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:33626_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 192943 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.s.SolrIndexSearcher Opening [Searcher@52409000[collection1_shard2_replica_t45] main]
   [junit4]   2> 192943 INFO  (recoveryExecutor-111-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 192945 INFO  (searcherExecutor-322-thread-1-processing-n:127.0.0.1:34723_ x:collection1_shard2_replica_t45 s:shard2 c:collection1 r:core_node46) [n:127.0.0.1:34723_ c:collection1 s:shard2 r:core_node46 x:collection1_shard2_replica_t45] o.a.s.c.SolrCore [collection1_shard2_replica_t45] Registered new searcher Searcher@52409000[collection1_shard2_replica_t45] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 192966 INFO  (TEST-ShardSplitTest.test-seed#[93B52C3220965026]) [n:127.0.0.1:33626_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33626_
   [junit4]   2> 192979 INFO  (zkCallback-96-thread-1) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 192982 INFO  (zkCallback-113-thread-1-processing-n:127.0.0.1:34723_) [n:127.0.0.1:34723_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 192999 INFO  (zkCallback-88-thread-4-processing-n:127.0.0.1:35745_) [n:127.0.0.1:35745_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
   [junit4]   2> 193109 INFO  (zkCallback-120-thread-1-processing-n:127.0.0.1:33626_) [n:127.0.0.1:33626_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper.

[...truncated too long message...]

1:38501_ c:collection1 s:shard1_1 r:core_node60 x:collection1_shard1_1_replica0] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@605f543c: rootName = null, domain = solr.core.collection1.shard1_1.replica0, service url = null, agent id = null] for registry solr.core.collection1.shard1_1.replica0 / com.codahale.metrics.MetricRegistry@2cc72b4d
   [junit4]   2> 825146 INFO  (zkCallback-335-thread-1-processing-n:127.0.0.1:40999_) [n:127.0.0.1:40999_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_t47] o.a.s.c.ZkController collection1_shard1_replica_t47 stopping background replication from leader
   [junit4]   2> 825147 INFO  (zkCallback-335-thread-1-processing-n:127.0.0.1:40999_) [n:127.0.0.1:40999_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_t47] o.a.s.c.ShardLeaderElectionContext New leader does not have old tlog to replay
   [junit4]   2> 825148 INFO  (zkCallback-335-thread-1-processing-n:127.0.0.1:40999_) [n:127.0.0.1:40999_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_t47] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40999/collection1_shard1_replica_t47/ shard1
   [junit4]   2> 825190 INFO  (zkCallback-328-thread-2-processing-n:127.0.0.1:38501_) [n:127.0.0.1:38501_ c:collection1 s:shard1_1 r:core_node60 x:collection1_shard1_1_replica0] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1_1.leader, tag=107829361
   [junit4]   2> 825262 INFO  (zkCallback-335-thread-3-processing-n:127.0.0.1:40999_) [n:127.0.0.1:40999_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 825290 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@74c801ae{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 825290 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2647e0a6{/,null,UNAVAILABLE}
   [junit4]   2> 825291 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.c.ChaosMonkey monkey: stop jetty! 40999
   [junit4]   2> 825291 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=917176294
   [junit4]   2> 825291 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 825291 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1118a46c: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@392ca992
   [junit4]   2> 825310 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 825311 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2353c2cd: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@81b2ab8
   [junit4]   2> 825313 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> 825313 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@488e4a33: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@3b2c1ec4
   [junit4]   2> 825315 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:40999_
   [junit4]   2> 825316 INFO  (zkCallback-311-thread-2) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 825316 INFO  (zkCallback-335-thread-3-processing-n:127.0.0.1:40999_) [n:127.0.0.1:40999_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
   [junit4]   2> 825316 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
   [junit4]   2> 825317 INFO  (coreCloseExecutor-1022-thread-1) [n:127.0.0.1:40999_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_t47] o.a.s.c.SolrCore [collection1_shard1_replica_t47]  CLOSING SolrCore org.apache.solr.core.SolrCore@7db59ed4
   [junit4]   2> 825317 INFO  (coreCloseExecutor-1022-thread-1) [n:127.0.0.1:40999_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_t47] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.replica_t47, tag=2109054676
   [junit4]   2> 825317 INFO  (coreCloseExecutor-1022-thread-1) [n:127.0.0.1:40999_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_t47] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7848b818: rootName = null, domain = solr.core.collection1.shard1.replica_t47, service url = null, agent id = null] for registry solr.core.collection1.shard1.replica_t47 / com.codahale.metrics.MetricRegistry@240d5590
   [junit4]   2> 825352 INFO  (coreCloseExecutor-1022-thread-1) [n:127.0.0.1:40999_ c:collection1 s:shard1 r:core_node48 x:collection1_shard1_replica_t47] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=2109054676
   [junit4]   2> 825371 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.c.Overseer Overseer (id=99027497993502738-127.0.0.1:40999_-n_0000000004) closing
   [junit4]   2> 825371 INFO  (OverseerStateUpdate-99027497993502738-127.0.0.1:40999_-n_0000000004) [n:127.0.0.1:40999_    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:40999_
   [junit4]   2> 825391 WARN  (OverseerAutoScalingTriggerThread-99027497993502738-127.0.0.1:40999_-n_0000000004) [n:127.0.0.1:40999_    ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
   [junit4]   2> 825393 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.e.j.s.AbstractConnector Stopped ServerConnector@7b804cd2{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 825394 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4805daf0{/,null,UNAVAILABLE}
   [junit4]   2> 825395 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.c.ChaosMonkey monkey: stop jetty! 44976
   [junit4]   2> 825395 ERROR (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 825420 INFO  (TEST-ShardSplitTest.testSplitAfterFailedSplit-seed#[93B52C3220965026]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33760 33760
   [junit4]   2> 825541 INFO  (Thread-338) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:33760 33760
   [junit4]   2> 825562 WARN  (Thread-338) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/aliases.json
   [junit4]   2> 	5	/solr/security.json
   [junit4]   2> 	5	/solr/configs/conf1
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/clusterprops.json
   [junit4]   2> 	7	/solr/clusterstate.json
   [junit4]   2> 	5	/solr/autoscaling.json
   [junit4]   2> 	4	/solr/collections/collection1/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	7	/solr/live_nodes
   [junit4]   2> 	7	/solr/collections
   [junit4]   2> 	5	/solr/autoscaling/events/.auto_add_replicas
   [junit4]   2> 	5	/solr/overseer/collection-queue-work
   [junit4]   2> 	5	/solr/overseer/queue-work
   [junit4]   2> 	4	/solr/overseer/queue
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.ShardSplitTest_93B52C3220965026-001
   [junit4]   2> Nov 18, 2017 9:17:07 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=ar-DZ, timezone=America/Caracas
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=108849920,total=506462208
   [junit4]   2> NOTE: All tests run in this JVM: [ShardRoutingCustomTest, TestRemoteStreaming, AddSchemaFieldsUpdateProcessorFactoryTest, OpenCloseCoreStressTest, TestConfigOverlay, TestTrackingShardHandlerFactory, BooleanFieldTest, TestFastWriter, SuggestComponentContextFilterQueryTest, ShardSplitTest]
   [junit4] Completed [87/749 (1!)] on J1 in 657.36s, 5 tests, 1 error <<< FAILURES!

[...truncated 49535 lines...]

[JENKINS] Lucene-Solr-Tests-7.x - Build # 247 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/247/

4 tests failed.
FAILED:  org.apache.solr.cloud.TestPullReplica.testKillLeader

Error Message:
Replica core_node4 not up to date after 10 seconds expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: Replica core_node4 not up to date after 10 seconds expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([F0E873C7465A2454:B9FE877324E1B002]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.TestPullReplica.waitForNumDocsInAllReplicas(TestPullReplica.java:508)
	at org.apache.solr.cloud.TestPullReplica.doTestNoLeader(TestPullReplica.java:456)
	at org.apache.solr.cloud.TestPullReplica.testKillLeader(TestPullReplica.java:289)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)


FAILED:  org.apache.solr.core.TestLazyCores.testNoCommit

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([F0E873C7465A2454:2F88D2168D7D47F1]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:901)
	at org.apache.solr.core.TestLazyCores.check10(TestLazyCores.java:847)
	at org.apache.solr.core.TestLazyCores.testNoCommit(TestLazyCores.java:829)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result[@numFound='10']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">0</int><lst name="params"><str name="q">*:*</str></lst></lst><result name="response" numFound="0" start="0"></result>
</response>

	request was:q=*:*
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:894)
	... 41 more


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores:     1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([F0E873C7465A2454]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.core.TestLazyCores

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)         at java.lang.Thread.run(Thread.java:748)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
        at java.lang.Thread.run(Thread.java:748)
	at __randomizedtesting.SeedInfo.seed([F0E873C7465A2454]:0)




Build Log:
[...truncated 11720 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestPullReplica
   [junit4]   2> 290433 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
   [junit4]   2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/init-core-data-001
   [junit4]   2> 290433 WARN  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=3 numCloses=3
   [junit4]   2> 290539 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
   [junit4]   2> 290541 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
   [junit4]   2> 290687 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001
   [junit4]   2> 290687 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 290701 INFO  (Thread-86) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 290702 INFO  (Thread-86) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 290745 ERROR (Thread-86) [    ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
   [junit4]   2> 290811 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:44818
   [junit4]   2> 290951 INFO  (jetty-launcher-104-thread-1) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 290951 INFO  (jetty-launcher-104-thread-2) [    ] o.e.j.s.Server jetty-9.3.20.v20170531
   [junit4]   2> 290972 INFO  (jetty-launcher-104-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@30bf189a{/solr,null,AVAILABLE}
   [junit4]   2> 291056 INFO  (jetty-launcher-104-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6928c715{/solr,null,AVAILABLE}
   [junit4]   2> 291081 INFO  (jetty-launcher-104-thread-1) [    ] o.e.j.s.AbstractConnector Started ServerConnector@68078061{SSL,[ssl, http/1.1]}{127.0.0.1:45743}
   [junit4]   2> 291083 INFO  (jetty-launcher-104-thread-1) [    ] o.e.j.s.Server Started @306995ms
   [junit4]   2> 291082 INFO  (jetty-launcher-104-thread-2) [    ] o.e.j.s.AbstractConnector Started ServerConnector@6018542f{SSL,[ssl, http/1.1]}{127.0.0.1:35285}
   [junit4]   2> 291083 INFO  (jetty-launcher-104-thread-2) [    ] o.e.j.s.Server Started @306995ms
   [junit4]   2> 291083 INFO  (jetty-launcher-104-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45743}
   [junit4]   2> 291083 INFO  (jetty-launcher-104-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35285}
   [junit4]   2> 291096 ERROR (jetty-launcher-104-thread-2) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 291096 ERROR (jetty-launcher-104-thread-1) [    ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
   [junit4]   2> 291169 INFO  (jetty-launcher-104-thread-2) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 291169 INFO  (jetty-launcher-104-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr™ version 7.2.0
   [junit4]   2> 291190 INFO  (jetty-launcher-104-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 291190 INFO  (jetty-launcher-104-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 291190 INFO  (jetty-launcher-104-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-19T17:39:41.721Z
   [junit4]   2> 291190 INFO  (jetty-launcher-104-thread-2) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 291190 INFO  (jetty-launcher-104-thread-2) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 291190 INFO  (jetty-launcher-104-thread-2) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2017-11-19T17:39:41.721Z
   [junit4]   2> 291338 INFO  (jetty-launcher-104-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 291353 INFO  (jetty-launcher-104-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 291508 INFO  (jetty-launcher-104-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44818/solr
   [junit4]   2> 291600 INFO  (jetty-launcher-104-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44818/solr
   [junit4]   2> 293580 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 293614 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35285_solr
   [junit4]   2> 293615 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.c.Overseer Overseer (id=99032313445285892-127.0.0.1:35285_solr-n_0000000000) starting
   [junit4]   2> 293715 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35285_solr
   [junit4]   2> 293740 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 293742 INFO  (jetty-launcher-104-thread-1) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 293813 INFO  (jetty-launcher-104-thread-1) [n:127.0.0.1:45743_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 293814 INFO  (jetty-launcher-104-thread-1) [n:127.0.0.1:45743_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45743_solr
   [junit4]   2> 293845 INFO  (zkCallback-112-thread-2-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 293920 INFO  (zkCallback-116-thread-1-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 295034 INFO  (jetty-launcher-104-thread-1) [n:127.0.0.1:45743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45743.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 295055 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35285.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 295217 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35285.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 295218 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35285.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 295313 INFO  (jetty-launcher-104-thread-1) [n:127.0.0.1:45743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45743.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 295313 INFO  (jetty-launcher-104-thread-1) [n:127.0.0.1:45743_solr    ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45743.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 295219 INFO  (jetty-launcher-104-thread-2) [n:127.0.0.1:35285_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/.
   [junit4]   2> 295366 INFO  (jetty-launcher-104-thread-1) [n:127.0.0.1:45743_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/.
   [junit4]   2> 296037 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x15fd55d48310007, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> 296120 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
   [junit4]   2> 296121 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44818/solr ready
   [junit4]   2> 296156 INFO  (SUITE-TestPullReplica-seed#[F0E873C7465A2454]-worker) [    ] o.a.s.c.TestPullReplica Using legacyCloud?: false
   [junit4]   2> 296852 INFO  (qtp329021367-582) [n:127.0.0.1:35285_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 296861 INFO  (qtp329021367-582) [n:127.0.0.1:35285_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=9
   [junit4]   2> 297028 INFO  (TEST-TestPullReplica.testCreateDelete-seed#[F0E873C7465A2454]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testCreateDelete {seed=[F0E873C7465A2454:EBF8828BB0AADC64]}
   [junit4]   2> 297362 INFO  (OverseerThreadFactory-357-thread-1-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_create_delete
   [junit4]   2> 297399 WARN  (OverseerThreadFactory-357-thread-1-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_create_delete is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
   [junit4]   2> 297812 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_n1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35285/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 297846 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_p2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45743/solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 297848 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_p3",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35285/solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 297849 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard1_replica_p4",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45743/solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 297863 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_n5",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35285/solr",
   [junit4]   2>   "type":"NRT",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 297865 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_p6",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45743/solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 297865 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_p7",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:35285/solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 297882 INFO  (OverseerStateUpdate-99032313445285892-127.0.0.1:35285_solr-n_0000000000) [n:127.0.0.1:35285_solr    ] o.a.s.c.o.SliceMutator createReplica() {
   [junit4]   2>   "operation":"ADDREPLICA",
   [junit4]   2>   "collection":"pull_replica_test_create_delete",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "core":"pull_replica_test_create_delete_shard2_replica_p10",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "base_url":"https://127.0.0.1:45743/solr",
   [junit4]   2>   "type":"PULL",
   [junit4]   2>   "waitForFinalState":"false"} 
   [junit4]   2> 298250 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p3&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 298250 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 298251 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node15&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p7&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 298263 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_n1&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 298559 INFO  (zkCallback-112-thread-2-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 298842 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_n5&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=NRT
   [junit4]   2> 298918 INFO  (zkCallback-112-thread-2-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 299171 INFO  (zkCallback-112-thread-2-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 299274 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p2&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 299275 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr    ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
   [junit4]   2> 299586 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node16&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p10&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 299587 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node12&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p4&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 299613 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node14&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_p6&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=PULL
   [junit4]   2> 299683 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 299720 INFO  (zkCallback-112-thread-2-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 299732 INFO  (zkCallback-116-thread-1-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 299753 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 300068 INFO  (zkCallback-116-thread-2-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 300098 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 300126 INFO  (zkCallback-112-thread-2-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 300130 INFO  (zkCallback-116-thread-1-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 300131 INFO  (zkCallback-116-thread-2-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 300141 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p7] Schema name=minimal
   [junit4]   2> 300193 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p3] Schema name=minimal
   [junit4]   2> 300209 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_n1] Schema name=minimal
   [junit4]   2> 300228 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 300228 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p3' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 300229 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35285.solr.core.pull_replica_test_create_delete.shard1.replica_p3' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 300229 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 300229 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_p3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_p3/data/]
   [junit4]   2> 300249 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 300249 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p7' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 300264 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35285.solr.core.pull_replica_test_create_delete.shard2.replica_p7' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 300264 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 300264 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_n1' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 300264 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 300264 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p7] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_p7], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_p7/data/]
   [junit4]   2> 300265 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35285.solr.core.pull_replica_test_create_delete.shard1.replica_n1' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 300265 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 300265 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/pull_replica_test_create_delete_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/./pull_replica_test_create_delete_shard1_replica_n1/data/]
   [junit4]   2> 300267 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 300342 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_n5] Schema name=minimal
   [junit4]   2> 300344 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 300344 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_n5' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 300358 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35285.solr.core.pull_replica_test_create_delete.shard2.replica_n5' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_n5') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 300358 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 300358 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_n5] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/pull_replica_test_create_delete_shard2_replica_n5], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node2/./pull_replica_test_create_delete_shard2_replica_n5/data/]
   [junit4]   2> 300955 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 301268 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 301393 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 301414 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p4] Schema name=minimal
   [junit4]   2> 301479 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 301479 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 301497 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.s.SolrIndexSearcher Opening [Searcher@10b1926f[pull_replica_test_create_delete_shard1_replica_p3] main]
   [junit4]   2> 301502 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard1_replica_p2] Schema name=minimal
   [junit4]   2> 301504 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 301504 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p2' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 301540 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.2.0
   [junit4]   2> 301550 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 301558 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45743.solr.core.pull_replica_test_create_delete.shard1.replica_p2' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 301558 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 301558 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p2/data/]
   [junit4]   2> 301561 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 301561 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard1_replica_p4' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 301561 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45743.solr.core.pull_replica_test_create_delete.shard1.replica_p4' (registry 'solr.core.pull_replica_test_create_delete.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 301561 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 301562 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard1_replica_p4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/pull_replica_test_create_delete_shard1_replica_p4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/./pull_replica_test_create_delete_shard1_replica_p4/data/]
   [junit4]   2> 301647 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 301706 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 301794 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p3 starting background replication from leader
   [junit4]   2> 301795 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 301812 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p6] Schema name=minimal
   [junit4]   2> 302001 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 302001 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p6' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 302018 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45743.solr.core.pull_replica_test_create_delete.shard2.replica_p6' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 302018 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 302018 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p6] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p6], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p6/data/]
   [junit4]   2> 302055 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema [pull_replica_test_create_delete_shard2_replica_p10] Schema name=minimal
   [junit4]   2> 302402 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 302402 INFO  (qtp329021367-581) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 302423 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 302423 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_create_delete_shard2_replica_p10' using configuration from collection pull_replica_test_create_delete, trusted=true
   [junit4]   2> 302423 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45743.solr.core.pull_replica_test_create_delete.shard2.replica_p10' (registry 'solr.core.pull_replica_test_create_delete.shard2.replica_p10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7e6c5bba
   [junit4]   2> 302423 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
   [junit4]   2> 302423 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [[pull_replica_test_create_delete_shard2_replica_p10] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/pull_replica_test_create_delete_shard2_replica_p10], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestPullReplica_F0E873C7465A2454-001/tempDir-001/node1/./pull_replica_test_create_delete_shard2_replica_p10/data/]
   [junit4]   2> 302436 INFO  (searcherExecutor-362-thread-1-processing-n:127.0.0.1:35285_solr x:pull_replica_test_create_delete_shard1_replica_p3 s:shard1 c:pull_replica_test_create_delete r:core_node11) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node11 x:pull_replica_test_create_delete_shard1_replica_p3] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p3] Registered new searcher Searcher@10b1926f[pull_replica_test_create_delete_shard1_replica_p3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 302879 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 302879 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 302893 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.s.SolrIndexSearcher Opening [Searcher@59886f81[pull_replica_test_create_delete_shard2_replica_p7] main]
   [junit4]   2> 302896 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 302896 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 302898 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 302941 INFO  (searcherExecutor-363-thread-1-processing-n:127.0.0.1:35285_solr x:pull_replica_test_create_delete_shard2_replica_p7 s:shard2 c:pull_replica_test_create_delete r:core_node15) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p7] Registered new searcher Searcher@59886f81[pull_replica_test_create_delete_shard2_replica_p7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 302950 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p7 starting background replication from leader
   [junit4]   2> 302950 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 303152 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 303152 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 303153 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 303153 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 303154 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@66bf0f07[pull_replica_test_create_delete_shard1_replica_n1] main]
   [junit4]   2> 303082 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 303168 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 303205 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 303205 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 303287 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 303287 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 303289 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.s.SolrIndexSearcher Opening [Searcher@3a479c2e[pull_replica_test_create_delete_shard2_replica_n5] main]
   [junit4]   2> 303298 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 303298 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584517028332437504
   [junit4]   2> 303334 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 303334 INFO  (qtp329021367-583) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 303372 INFO  (searcherExecutor-364-thread-1-processing-n:127.0.0.1:35285_solr x:pull_replica_test_create_delete_shard1_replica_n1 s:shard1 c:pull_replica_test_create_delete r:core_node8) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_n1] Registered new searcher Searcher@66bf0f07[pull_replica_test_create_delete_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 303372 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 303373 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 303373 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 303373 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1584517028411080704
   [junit4]   2> 303428 INFO  (searcherExecutor-365-thread-1-processing-n:127.0.0.1:35285_solr x:pull_replica_test_create_delete_shard2_replica_n5 s:shard2 c:pull_replica_test_create_delete r:core_node13) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_n5] Registered new searcher Searcher@3a479c2e[pull_replica_test_create_delete_shard2_replica_n5] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 303447 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 303447 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 303447 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:35285/solr/pull_replica_test_create_delete_shard1_replica_n1/
   [junit4]   2> 303447 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 303447 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:35285/solr/pull_replica_test_create_delete_shard1_replica_n1/ has no replicas
   [junit4]   2> 303447 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 303463 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 303464 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 303464 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:35285/solr/pull_replica_test_create_delete_shard2_replica_n5/
   [junit4]   2> 303464 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 303464 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.SyncStrategy https://127.0.0.1:35285/solr/pull_replica_test_create_delete_shard2_replica_n5/ has no replicas
   [junit4]   2> 303464 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
   [junit4]   2> 303482 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:35285/solr/pull_replica_test_create_delete_shard1_replica_n1/ shard1
   [junit4]   2> 303484 INFO  (zkCallback-112-thread-1-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303513 INFO  (zkCallback-116-thread-2-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303513 INFO  (zkCallback-116-thread-1-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303537 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:35285/solr/pull_replica_test_create_delete_shard2_replica_n5/ shard2
   [junit4]   2> 303681 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 303681 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 303697 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@6c74872[pull_replica_test_create_delete_shard1_replica_p4] main]
   [junit4]   2> 303703 INFO  (zkCallback-112-thread-1-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303709 INFO  (zkCallback-116-thread-2-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303709 INFO  (zkCallback-116-thread-1-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303717 INFO  (indexFetcher-388-thread-1) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node15 x:pull_replica_test_create_delete_shard2_replica_p7] o.a.s.h.IndexFetcher Replica core_node13 is leader but it's state is down, skipping replication
   [junit4]   2> 303754 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 303754 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 303770 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 303770 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 303772 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7dfd33a[pull_replica_test_create_delete_shard1_replica_p2] main]
   [junit4]   2> 303829 INFO  (qtp329021367-576) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard1 r:core_node8 x:pull_replica_test_create_delete_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_n1&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5566
   [junit4]   2> 303843 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 303843 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 303843 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 303844 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 303844 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 303845 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 303846 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p4 starting background replication from leader
   [junit4]   2> 303846 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 303918 INFO  (searcherExecutor-371-thread-1-processing-n:127.0.0.1:45743_solr x:pull_replica_test_create_delete_shard1_replica_p4 s:shard1 c:pull_replica_test_create_delete r:core_node12) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p4] Registered new searcher Searcher@6c74872[pull_replica_test_create_delete_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 303936 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 303936 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 303937 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 303937 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 303939 INFO  (zkCallback-112-thread-1-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303940 INFO  (qtp1594860229-574) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node12&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard1_replica_p4&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=4352
   [junit4]   2> 303945 INFO  (zkCallback-116-thread-1-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303945 INFO  (zkCallback-116-thread-2-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 303955 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.s.SolrIndexSearcher Opening [Searcher@4be1c047[pull_replica_test_create_delete_shard2_replica_p10] main]
   [junit4]   2> 303956 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
   [junit4]   2> 303956 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
   [junit4]   2> 303956 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 303965 INFO  (qtp329021367-638) [n:127.0.0.1:35285_solr c:pull_replica_test_create_delete s:shard2 r:core_node13 x:pull_replica_test_create_delete_shard2_replica_n5] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node13&collection.configName=conf&newCollection=true&name=pull_replica_test_create_delete_shard2_replica_n5&action=CREATE&numShards=2&collection=pull_replica_test_create_delete&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5123
   [junit4]   2> 303968 INFO  (searcherExecutor-370-thread-1-processing-n:127.0.0.1:45743_solr x:pull_replica_test_create_delete_shard1_replica_p2 s:shard1 c:pull_replica_test_create_delete r:core_node9) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard1_replica_p2] Registered new searcher Searcher@7dfd33a[pull_replica_test_create_delete_shard1_replica_p2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 303968 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.ZkController pull_replica_test_create_delete_shard1_replica_p2 starting background replication from leader
   [junit4]   2> 303969 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 304027 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.ZkController pull_replica_test_create_delete_shard2_replica_p10 starting background replication from leader
   [junit4]   2> 304027 INFO  (qtp1594860229-573) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
   [junit4]   2> 303937 INFO  (indexFetcher-397-thread-1) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node12 x:pull_replica_test_create_delete_shard1_replica_p4] o.a.s.h.IndexFetcher Replica core_node8 is leader but it's state is down, skipping replication
   [junit4]   2> 304028 INFO  (searcherExecutor-377-thread-1-processing-n:127.0.0.1:45743_solr x:pull_replica_test_create_delete_shard2_replica_p10 s:shard2 c:pull_replica_test_create_delete r:core_node16) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node16 x:pull_replica_test_create_delete_shard2_replica_p10] o.a.s.c.SolrCore [pull_replica_test_create_delete_shard2_replica_p10] Registered new searcher Searcher@4be1c047[pull_replica_test_create_delete_shard2_replica_p10] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 304063 INFO  (zkCallback-112-thread-1-processing-n:127.0.0.1:35285_solr) [n:127.0.0.1:35285_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 304066 INFO  (zkCallback-116-thread-2-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 304066 INFO  (zkCallback-116-thread-1-processing-n:127.0.0.1:45743_solr) [n:127.0.0.1:45743_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_create_delete/state.json] for collection [pull_replica_test_create_delete] has occurred - updating... (live nodes size: [2])
   [junit4]   2> 304087 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
   [junit4]   2> 304087 INFO  (qtp1594860229-570) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
   [junit4]   2> 304088 INFO  (indexFetcher-402-thread-1) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard1 r:core_node9 x:pull_replica_test_create_delete_shard1_replica_p2] o.a.s.h.IndexFetcher Updated masterUrl to https://127.0.0.1:35285/solr/pull_replica_test_create_delete_shard1_replica_n1/
   [junit4]   2> 304139 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 304139 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_shard2_replica_p6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 304175 INFO  (qtp1594860229-567) [n:127.0.0.1:45743_solr c:pull_replica_test_create_delete s:shard2 r:core_node14 x:pull_replica_test_create_delete_

[...truncated too long message...]

collection2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@27b91302: rootName = null, domain = solr.core.collection2, service url = null, agent id = null] for registry solr.core.collection2 / com.codahale.metrics.MetricRegistry@26381cd3
   [junit4]   2> 3188947 INFO  (TEST-TestLazyCores.testLazySearch-seed#[F0E873C7465A2454]) [    x:collection4] o.a.s.SolrTestCaseJ4 ###Ending testLazySearch
   [junit4]   2> 3188947 INFO  (SUITE-TestLazyCores-seed#[F0E873C7465A2454]-worker) [    x:collection4] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 3188947 INFO  (SUITE-TestLazyCores-seed#[F0E873C7465A2454]-worker) [    x:collection4] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1809446273
   [junit4]   2> 3188947 INFO  (SUITE-TestLazyCores-seed#[F0E873C7465A2454]-worker) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
   [junit4]   2> 3188947 INFO  (SUITE-TestLazyCores-seed#[F0E873C7465A2454]-worker) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
   [junit4]   2> 3188947 INFO  (SUITE-TestLazyCores-seed#[F0E873C7465A2454]-worker) [    x:collection4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
   [junit4]   2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.core.TestLazyCores_F0E873C7465A2454-001
   [junit4]   2> Nov 19, 2017 6:27:58 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Nov 19, 2017 6:28:18 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]   2>    1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> Nov 19, 2017 6:28:18 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2> Nov 19, 2017 6:28:21 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:748)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene70): {v_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), v_s=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), v_f=BlockTreeOrds(blocksize=128)}, docValues:{}, maxPointsInLeafNode=396, maxMBSortInHeap=5.062073501111163, sim=RandomSimilarity(queryNorm=true): {}, locale=en-GB, timezone=Europe/Luxembourg
   [junit4]   2> NOTE: Linux 4.4.0-83-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=2,free=300814712,total=530579456
   [junit4]   2> NOTE: All tests run in this JVM: [TriLevelCompositeIdRoutingTest, SuggesterFSTTest, OverseerRolesTest, TestAnalyzedSuggestions, SOLR749Test, ZkSolrClientTest, TestHighFrequencyDictionaryFactory, TestJsonFacets, TestInitQParser, NotRequiredUniqueKeyTest, TestDeleteCollectionOnDownNodes, TestRetrieveFieldsOptimizer, MBeansHandlerTest, HdfsDirectoryFactoryTest, WrapperMergePolicyFactoryTest, RegexBoostProcessorTest, DocValuesMultiTest, TestCloudRecovery, TestSimpleTrackingShardHandler, TestStressInPlaceUpdates, TestAddFieldRealTimeGet, SuggestComponentContextFilterQueryTest, SynonymTokenizerTest, TestPrepRecovery, TestLegacyFieldReuse, SolrSlf4jReporterTest, TestLockTree, TestLuceneMatchVersion, SimpleMLTQParserTest, PreAnalyzedFieldTest, TestStressLucene, TestIntervalFaceting, SuggesterTSTTest, LukeRequestHandlerTest, ConvertedLegacyTest, BasicAuthStandaloneTest, TermVectorComponentDistributedTest, MultiThreadedOCPTest, TestConfigReload, ConjunctionSolrSpellCheckerTest, ZkCLITest, SpatialRPTFieldTypeTest, TestRandomFlRTGCloud, ReplicaListTransformerTest, SolrMetricReporterTest, ZkStateReaderTest, TestAtomicUpdateErrorCases, MetricsConfigTest, CdcrRequestHandlerTest, BadIndexSchemaTest, BooleanFieldTest, TestStressRecovery, TestDocTermOrds, NumberUtilsTest, TestLeaderElectionZkExpiry, TestNonDefinedSimilarityFactory, PingRequestHandlerTest, PrimUtilsTest, CloudExitableDirectoryReaderTest, TestQueryWrapperFilter, TestManagedSynonymFilterFactory, SignatureUpdateProcessorFactoryTest, HdfsCollectionsAPIDistributedZkTest, TestSQLHandlerNonCloud, BinaryUpdateRequestHandlerTest, TestDocSet, FullHLLTest, TestPushWriter, SampleTest, DeleteShardTest, MergeStrategyTest, CoreSorterTest, ShowFileRequestHandlerTest, TestConfigSets, AnalyticsQueryTest, TestConfigSetsAPIExclusivity, TemplateUpdateProcessorTest, DateRangeFieldTest, TestJettySolrRunner, TestSerializedLuceneMatchVersion, ManagedSchemaRoundRobinCloudTest, DistributedExpandComponentTest, TestSolr4Spatial2, TestSurroundQueryParser, TestSearcherReuse, RollingRestartTest, TestReqParamsAPI, BigEndianAscendingWordSerializerTest, CollectionTooManyReplicasTest, TestFieldTypeCollectionResource, TestManagedSchema, TestSolrXml, RegexBytesRefFilterTest, TestOmitPositions, DistributedVersionInfoTest, DistributedIntervalFacetingTest, ClassificationUpdateProcessorFactoryTest, BufferStoreTest, TestWriterPerf, ClusterStateUpdateTest, HdfsRecoveryZkTest, TestExceedMaxTermLength, TriggerIntegrationTest, TestLazyCores]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLazyCores -Dtests.seed=F0E873C7465A2454 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=en-GB -Dtests.timezone=Europe/Luxembourg -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestLazyCores (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.core.TestLazyCores: 
   [junit4]    >    1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F0E873C7465A2454]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=8870, name=searcherExecutor-3806-thread-1, state=WAITING, group=TGRP-TestLazyCores]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
   [junit4]    >         at java.lang.Thread.run(Thread.java:748)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F0E873C7465A2454]:0)
   [junit4] Completed [322/749 (2!)] on J1 in 52.35s, 9 tests, 3 errors <<< FAILURES!

[...truncated 48652 lines...]