You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/06/13 05:21:51 UTC
[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 472 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/472/
1 tests failed.
FAILED: org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates
Error Message:
Timeout while trying to assert number of documents @ source_collection
Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ source_collection
at __randomizedtesting.SeedInfo.seed([787D1CF68CEFFA2E:AB744CE8C97C66B9]:0)
at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:277)
at org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:233)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.AssertionError: expected:<560> but was:<570>
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotEquals(Assert.java:834)
at org.junit.Assert.assertEquals(Assert.java:645)
at org.junit.Assert.assertEquals(Assert.java:631)
at org.apache.solr.cloud.cdcr.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:268)
... 42 more
Build Log:
[...truncated 13515 lines...]
[junit4] Suite: org.apache.solr.cloud.cdcr.CdcrReplicationHandlerTest
[junit4] 2> 118287 INFO (SUITE-CdcrReplicationHandlerTest-seed#[787D1CF68CEFFA2E]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/server/solr/configsets/_default/conf'
[junit4] 2> 118287 INFO (SUITE-CdcrReplicationHandlerTest-seed#[787D1CF68CEFFA2E]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/data-dir-10-001
[junit4] 2> 118287 WARN (SUITE-CdcrReplicationHandlerTest-seed#[787D1CF68CEFFA2E]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=19 numCloses=19
[junit4] 2> 118287 INFO (SUITE-CdcrReplicationHandlerTest-seed#[787D1CF68CEFFA2E]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 118289 INFO (SUITE-CdcrReplicationHandlerTest-seed#[787D1CF68CEFFA2E]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 118289 INFO (SUITE-CdcrReplicationHandlerTest-seed#[787D1CF68CEFFA2E]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 118289 INFO (SUITE-CdcrReplicationHandlerTest-seed#[787D1CF68CEFFA2E]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_uhb/
[junit4] 2> 118291 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 118291 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
[junit4] 2> 118291 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 118391 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer start zk server on port: 44061
[junit4] 2> 118391 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:44061
[junit4] 2> 118391 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:44061
[junit4] 2> 118391 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 44061
[junit4] 2> 118393 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 118395 INFO (zkConnectionManagerCallback-606-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 118395 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 118398 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 118403 INFO (zkConnectionManagerCallback-608-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 118403 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 118404 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 118405 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 118406 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 118407 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 118407 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 118408 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 118409 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 118409 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 118410 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 118411 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 118411 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 118412 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testFullReplication
[junit4] 2> 118585 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 118585 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 118585 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 118585 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 118586 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 118586 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 118586 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 118586 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2b87d818{/_uhb,null,AVAILABLE}
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@603a00d8{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:39607}
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.Server Started @118617ms
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, hostContext=/_uhb, hostPort=39607, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-001/cores}
[junit4] 2> 118587 ERROR (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 118587 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-13T03:15:13.066Z
[junit4] 2> 118588 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 118589 INFO (zkConnectionManagerCallback-610-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 118589 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 118690 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 118691 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-001/solr.xml
[junit4] 2> 118696 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 118696 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 118698 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 119223 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 119225 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@526ae93f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 119226 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@526ae93f[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 119232 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@19b2be53[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 119232 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@19b2be53[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 119236 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44061/solr
[junit4] 2> 119237 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 119239 INFO (zkConnectionManagerCallback-621-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 119239 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 119343 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 119346 INFO (zkConnectionManagerCallback-623-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 119346 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 119421 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:39607__uhb
[junit4] 2> 119421 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.c.Overseer Overseer (id=73921418611458052-127.0.0.1:39607__uhb-n_0000000000) starting
[junit4] 2> 119425 INFO (OverseerStateUpdate-73921418611458052-127.0.0.1:39607__uhb-n_0000000000) [n:127.0.0.1:39607__uhb ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:39607__uhb
[junit4] 2> 119425 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39607__uhb
[junit4] 2> 119426 INFO (zkCallback-622-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 119427 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 119428 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 119448 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 119466 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 119473 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 119473 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 119474 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:39607__uhb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-001/cores
[junit4] 2> 119575 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 119575 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 119575 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 119575 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 119580 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 119580 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 119580 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 119580 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2c2ae931{/_uhb,null,AVAILABLE}
[junit4] 2> 119581 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1728806{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:34777}
[junit4] 2> 119581 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.Server Started @119612ms
[junit4] 2> 119581 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard2, hostContext=/_uhb, hostPort=34777, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-002/cores}
[junit4] 2> 119582 ERROR (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 119582 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 119582 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 119582 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 119582 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 119582 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-13T03:15:14.061Z
[junit4] 2> 119585 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 119588 INFO (zkConnectionManagerCallback-637-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 119588 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 119689 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 119689 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-002/solr.xml
[junit4] 2> 119694 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 119694 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 119695 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 120299 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 120301 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@72eac867[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 120301 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@72eac867[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 120304 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@f8e4309[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 120304 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@f8e4309[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 120305 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44061/solr
[junit4] 2> 120305 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 120308 INFO (zkConnectionManagerCallback-648-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 120308 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 120413 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 120416 INFO (zkConnectionManagerCallback-650-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 120416 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 120420 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 120423 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.ZkController Publish node=127.0.0.1:34777__uhb as DOWN
[junit4] 2> 120424 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 120424 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34777__uhb
[junit4] 2> 120426 INFO (zkCallback-622-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 120426 INFO (zkCallback-649-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 120428 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 120428 WARN (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 120446 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 120467 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 120478 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 120478 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 120479 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [n:127.0.0.1:34777__uhb ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-002/cores
[junit4] 2> 120495 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 120496 INFO (zkConnectionManagerCallback-663-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 120496 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 120497 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 120498 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44061/solr ready
[junit4] 2> 120500 INFO (qtp807248154-1260) [n:127.0.0.1:34777__uhb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=tmp_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 120504 INFO (OverseerThreadFactory-630-thread-1-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb ] o.a.s.c.a.c.CreateCollectionCmd Create collection tmp_collection
[junit4] 2> 120607 WARN (OverseerThreadFactory-630-thread-1-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (tmp_collection) without cores.
[junit4] 2> 120609 INFO (qtp807248154-1260) [n:127.0.0.1:34777__uhb ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 120610 INFO (qtp807248154-1260) [n:127.0.0.1:34777__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=tmp_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&wt=javabin&version=2} status=0 QTime=110
[junit4] 2> 120611 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39607__uhb&action=ADDREPLICA&collection=tmp_collection&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 120613 INFO (OverseerCollectionConfigSetProcessor-73921418611458052-127.0.0.1:39607__uhb-n_0000000000) [n:127.0.0.1:39607__uhb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 120620 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 120621 INFO (qtp1186806583-1227) [n:127.0.0.1:39607__uhb ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 120621 INFO (qtp1186806583-1227) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 120625 INFO (OverseerThreadFactory-630-thread-2-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:39607__uhb for creating new replica of shard shard1 for collection tmp_collection
[junit4] 2> 120626 INFO (OverseerThreadFactory-630-thread-2-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 120630 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb x:tmp_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=tmp_collection_shard1_replica_n1&action=CREATE&collection=tmp_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 121642 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 121664 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=test
[junit4] 2> 121735 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 121788 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'tmp_collection_shard1_replica_n1' using configuration from configset conf1, trusted=true
[junit4] 2> 121788 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.tmp_collection.shard1.replica_n1' (registry 'solr.core.tmp_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 121795 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[tmp_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-001/cores/tmp_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-001/cores/tmp_collection_shard1_replica_n1/data/]
[junit4] 2> 121798 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=4, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 121907 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 121907 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 121908 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 121908 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 121909 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@573aee7e
[junit4] 2> 121913 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 121914 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 121914 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
[junit4] 2> 121917 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.h.CdcrBufferStateManager Created znode /collections/tmp_collection/cdcr/state/buffer
[junit4] 2> 121920 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.h.CdcrProcessStateManager Created znode /collections/tmp_collection/cdcr/state/process
[junit4] 2> 121930 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1669351988431683584
[junit4] 2> 121933 INFO (searcherExecutor-665-thread-1-processing-n:127.0.0.1:39607__uhb x:tmp_collection_shard1_replica_n1 c:tmp_collection s:shard1) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [tmp_collection_shard1_replica_n1] Registered new searcher autowarm time: 0 ms
[junit4] 2> 121936 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tmp_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 121936 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/tmp_collection/leaders/shard1
[junit4] 2> 121942 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 121942 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 121942 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:39607/_uhb/tmp_collection_shard1_replica_n1/
[junit4] 2> 121943 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 121943 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:39607/_uhb/tmp_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 121943 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/tmp_collection/leaders/shard1/leader after winning as /collections/tmp_collection/leader_elect/shard1/election/73921418611458052-core_node2-n_0000000000
[junit4] 2> 121943 INFO (zkCallback-622-thread-1) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ tmp_collection:shard1
[junit4] 2> 121946 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:39607/_uhb/tmp_collection_shard1_replica_n1/ shard1
[junit4] 2> 122048 INFO (zkCallback-622-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 122048 INFO (zkCallback-622-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 122049 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 122051 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=tmp_collection_shard1_replica_n1&action=CREATE&collection=tmp_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1420
[junit4] 2> 122053 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:39607__uhb&action=ADDREPLICA&collection=tmp_collection&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1442
[junit4] 2> 122054 INFO (qtp807248154-1259) [n:127.0.0.1:34777__uhb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:34777__uhb&action=ADDREPLICA&collection=tmp_collection&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 122055 INFO (OverseerCollectionConfigSetProcessor-73921418611458052-127.0.0.1:39607__uhb-n_0000000000) [n:127.0.0.1:39607__uhb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 122058 INFO (qtp807248154-1260) [n:127.0.0.1:34777__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 122059 INFO (qtp1186806583-1225) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.tmp_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 122060 INFO (qtp1186806583-1227) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 122062 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.tmp_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 122062 INFO (OverseerThreadFactory-630-thread-3-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:34777__uhb for creating new replica of shard shard1 for collection tmp_collection
[junit4] 2> 122063 INFO (OverseerThreadFactory-630-thread-3-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 122067 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=tmp_collection_shard1_replica_n3&action=CREATE&collection=tmp_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 122171 INFO (zkCallback-622-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 122171 INFO (zkCallback-622-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 122171 INFO (zkCallback-622-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 123078 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 123089 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.s.IndexSchema Schema name=test
[junit4] 2> 123157 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 123173 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.CoreContainer Creating SolrCore 'tmp_collection_shard1_replica_n3' using configuration from configset conf1, trusted=true
[junit4] 2> 123173 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.tmp_collection.shard1.replica_n3' (registry 'solr.core.tmp_collection.shard1.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@34ee9dba
[junit4] 2> 123173 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.SolrCore [[tmp_collection_shard1_replica_n3] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-002/cores/tmp_collection_shard1_replica_n3], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001/jetty-002/cores/tmp_collection_shard1_replica_n3/data/]
[junit4] 2> 123176 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=4, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 123253 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 123254 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 123255 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 123255 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 123255 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@35d5354f
[junit4] 2> 123258 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 123258 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 123258 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
[junit4] 2> 123260 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1669351989826289664
[junit4] 2> 123263 INFO (searcherExecutor-672-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.SolrCore [tmp_collection_shard1_replica_n3] Registered new searcher autowarm time: 0 ms
[junit4] 2> 123265 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tmp_collection/terms/shard1 to Terms{values={core_node2=0, core_node4=0}, version=1}
[junit4] 2> 123265 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/tmp_collection/leaders/shard1
[junit4] 2> 123267 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 r:core_node2 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.S.Request [tmp_collection_shard1_replica_n1] webapp=/_uhb path=/cdcr params={action=LASTPROCESSEDVERSION&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 123270 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.ZkController Core needs to recover:tmp_collection_shard1_replica_n3
[junit4] 2> 123270 INFO (updateExecutor-644-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 123270 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 123271 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy startupVersions is empty
[junit4] 2> 123272 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=tmp_collection_shard1_replica_n3&action=CREATE&collection=tmp_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1204
[junit4] 2> 123273 INFO (qtp1186806583-1228) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 r:core_node2 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.S.Request [tmp_collection_shard1_replica_n1] webapp=/_uhb path=/admin/ping params={wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 123273 INFO (qtp1186806583-1228) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 r:core_node2 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.S.Request [tmp_collection_shard1_replica_n1] webapp=/_uhb path=/admin/ping params={wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 123274 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[tmp_collection_shard1_replica_n3]
[junit4] 2> 123274 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 123274 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Publishing state of core [tmp_collection_shard1_replica_n3] as recovering, leader is [http://127.0.0.1:39607/_uhb/tmp_collection_shard1_replica_n1/] and I am [http://127.0.0.1:34777/_uhb/tmp_collection_shard1_replica_n3/]
[junit4] 2> 123275 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:39607/_uhb]; [WaitForState: action=PREPRECOVERY&core=tmp_collection_shard1_replica_n1&nodeName=127.0.0.1:34777__uhb&coreNodeName=core_node4&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 123275 INFO (qtp807248154-1259) [n:127.0.0.1:34777__uhb c:tmp_collection ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:34777__uhb&action=ADDREPLICA&collection=tmp_collection&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1221
[junit4] 2> 123277 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb x:tmp_collection_shard1_replica_n1 ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node4, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
[junit4] 2> 123277 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=tmp_collection, shard=shard1, thisCore=tmp_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:34777__uhb, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
[junit4] 2> "core":"tmp_collection_shard1_replica_n3",
[junit4] 2> "base_url":"http://127.0.0.1:34777/_uhb",
[junit4] 2> "node_name":"127.0.0.1:34777__uhb",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"}
[junit4] 2> 123277 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=tmp_collection, shard=shard1, thisCore=tmp_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:34777__uhb, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
[junit4] 2> "core":"tmp_collection_shard1_replica_n3",
[junit4] 2> "base_url":"http://127.0.0.1:34777/_uhb",
[junit4] 2> "node_name":"127.0.0.1:34777__uhb",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"}
[junit4] 2> 123277 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=tmp_collection, shard=shard1, thisCore=tmp_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:34777__uhb, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
[junit4] 2> "core":"tmp_collection_shard1_replica_n3",
[junit4] 2> "base_url":"http://127.0.0.1:34777/_uhb",
[junit4] 2> "node_name":"127.0.0.1:34777__uhb",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"}
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:down live:true
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:down live:true
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:down live:true
[junit4] 2> 123377 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: tmp_collection failOnTimeout: true timeout (sec):
[junit4] 2> 123378 INFO (zkCallback-622-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 123378 INFO (zkCallback-622-thread-4) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 123378 INFO (zkCallback-622-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 123378 INFO (zkCallback-649-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 123378 INFO (zkCallback-649-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 123378 INFO (zkCallback-649-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:recovering live:true
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:recovering live:true
[junit4] 2> 123379 INFO (watches-624-thread-2) [n:127.0.0.1:39607__uhb ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=tmp_collection, shard=shard1, thisCore=tmp_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:34777__uhb, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
[junit4] 2> "core":"tmp_collection_shard1_replica_n3",
[junit4] 2> "base_url":"http://127.0.0.1:34777/_uhb",
[junit4] 2> "node_name":"127.0.0.1:34777__uhb",
[junit4] 2> "state":"recovering",
[junit4] 2> "type":"NRT"}
[junit4] 2> 123379 INFO (watches-624-thread-1) [n:127.0.0.1:39607__uhb ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=tmp_collection, shard=shard1, thisCore=tmp_collection_shard1_replica_n1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:34777__uhb, coreNodeName=core_node4, onlyIfActiveCheckResult=false, nodeProps: core_node4:{
[junit4] 2> "core":"tmp_collection_shard1_replica_n3",
[junit4] 2> "base_url":"http://127.0.0.1:34777/_uhb",
[junit4] 2> "node_name":"127.0.0.1:34777__uhb",
[junit4] 2> "state":"recovering",
[junit4] 2> "type":"NRT"}
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:recovering live:true
[junit4] 2> 123380 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:34777__uhb&onlyIfLeaderActive=true&core=tmp_collection_shard1_replica_n1&coreNodeName=core_node4&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=102
[junit4] 2> 123880 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:39607/_uhb/tmp_collection_shard1_replica_n1/] - recoveringAfterStartup=[true]
[junit4] 2> 123880 WARN (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.PeerSyncWithLeader no frame of reference to tell if we've missed updates
[junit4] 2> 123880 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy PeerSync Recovery was not successful - trying replication.
[junit4] 2> 123880 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Starting Replication Recovery.
[junit4] 2> 123880 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Attempting to replicate from [http://127.0.0.1:39607/_uhb/tmp_collection_shard1_replica_n1/].
[junit4] 2> 123896 INFO (qtp807248154-1259) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.p.DistributedUpdateProcessor Ignoring commit while not ACTIVE - state: BUFFERING replay: false
[junit4] 2> 123896 INFO (qtp807248154-1259) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.S.Request [tmp_collection_shard1_replica_n3] webapp=/_uhb path=/update params={update.distrib=FROMLEADER&update.chain=cdcr-processor-chain&waitSearcher=true&openSearcher=false&commit=true&softCommit=false&distrib.from=http://127.0.0.1:39607/_uhb/tmp_collection_shard1_replica_n1/&commit_end_point=replicas&wt=javabin&version=2&expungeDeletes=false} status=0 QTime=0
[junit4] 2> 123898 INFO (qtp1186806583-1225) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 r:core_node2 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.S.Request [tmp_collection_shard1_replica_n1] webapp=/_uhb path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=17
[junit4] 2> 123900 INFO (qtp1186806583-1227) [n:127.0.0.1:39607__uhb c:tmp_collection s:shard1 r:core_node2 x:tmp_collection_shard1_replica_n1 ] o.a.s.c.S.Request [tmp_collection_shard1_replica_n1] webapp=/_uhb path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 123900 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 123900 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 123900 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 123900 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 123900 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 123904 INFO (searcherExecutor-672-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.SolrCore [tmp_collection_shard1_replica_n3] Registered new searcher autowarm time: 0 ms
[junit4] 2> 123904 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy No replay needed.
[junit4] 2> 123904 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Replication Recovery was successful.
[junit4] 2> 123904 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Registering as Active after recovery.
[junit4] 2> 123905 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Updating version bucket highest from index after successful recovery.
[junit4] 2> 123905 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1669351990502621184
[junit4] 2> 123906 INFO (recoveryExecutor-646-thread-1-processing-n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 c:tmp_collection s:shard1 r:core_node4) [n:127.0.0.1:34777__uhb c:tmp_collection s:shard1 r:core_node4 x:tmp_collection_shard1_replica_n3 ] o.a.s.c.RecoveryStrategy Finished recovery process, successful=[true]
[junit4] 2> 124009 INFO (zkCallback-649-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124009 INFO (zkCallback-622-thread-4) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124009 INFO (zkCallback-649-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124009 INFO (zkCallback-622-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124009 INFO (zkCallback-622-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124009 INFO (zkCallback-649-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:active live:true
[junit4] 1> no one is recoverying
[junit4] 2> 124010 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: tmp_collection
[junit4] 2> 124013 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 124015 INFO (zkConnectionManagerCallback-683-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 124015 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 124017 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 124019 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44061/solr ready
[junit4] 2> 124020 INFO (qtp807248154-1262) [n:127.0.0.1:34777__uhb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=tmp_collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 124025 INFO (OverseerCollectionConfigSetProcessor-73921418611458052-127.0.0.1:39607__uhb-n_0000000000) [n:127.0.0.1:39607__uhb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 124026 INFO (OverseerThreadFactory-630-thread-4-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb ] o.a.s.c.a.c.OverseerCollectionMessageHandler Executing Collection Cmd=action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true&deleteMetricsHistory=true, asyncId=null
[junit4] 2> 124028 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb x:tmp_collection_shard1_replica_n1 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.tmp_collection.shard1.replica_n1 tag=null
[junit4] 2> 124028 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb x:tmp_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1b41d4f2: rootName = null, domain = solr.core.tmp_collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.tmp_collection.shard1.replica_n1/com.codahale.metrics.MetricRegistry@1d94b058
[junit4] 2> 124028 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.tmp_collection.shard1.replica_n3 tag=null
[junit4] 2> 124028 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb x:tmp_collection_shard1_replica_n3 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3801c885: rootName = null, domain = solr.core.tmp_collection.shard1.replica_n3, service url = null, agent id = null] for registry solr.core.tmp_collection.shard1.replica_n3/com.codahale.metrics.MetricRegistry@70104d61
[junit4] 2> 124084 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.c.SolrCore [tmp_collection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@8714040
[junit4] 2> 124084 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.tmp_collection.shard1.replica_n1 tag=SolrCore@8714040
[junit4] 2> 124084 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.tmp_collection.shard1.leader tag=SolrCore@8714040
[junit4] 2> 124084 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ tmp_collection:shard1
[junit4] 2> 124084 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter.close() ... SKIPPED (unnecessary).
[junit4] 2> 124091 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.c.SolrCore [tmp_collection_shard1_replica_n3] CLOSING SolrCore org.apache.solr.core.SolrCore@9391ff5
[junit4] 2> 124091 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.tmp_collection.shard1.replica_n3 tag=SolrCore@9391ff5
[junit4] 2> 124091 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.tmp_collection.shard1.leader tag=SolrCore@9391ff5
[junit4] 2> 124091 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ tmp_collection:shard1
[junit4] 2> 124091 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter.close() ... SKIPPED (unnecessary).
[junit4] 2> 124094 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tmp_collection/terms/shard1 to Terms{values={core_node4=0}, version=2}
[junit4] 2> 124096 INFO (qtp1186806583-1226) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&deleteMetricsHistory=true&core=tmp_collection_shard1_replica_n1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=68
[junit4] 2> 124123 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/tmp_collection/terms/shard1 to Terms{values={}, version=3}
[junit4] 2> 124125 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.c.ShardLeaderElectionContextBase No version found for ephemeral leader parent node, won't remove previous leader registration.
[junit4] 2> 124126 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&deleteMetricsHistory=true&core=tmp_collection_shard1_replica_n3&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=97
[junit4] 2> 124230 INFO (zkCallback-649-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124230 INFO (zkCallback-649-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124230 INFO (zkCallback-622-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124230 INFO (zkCallback-649-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124230 INFO (zkCallback-622-thread-4) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124230 INFO (zkCallback-622-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDeleted path:/collections/tmp_collection/state.json] for collection [tmp_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 124243 INFO (qtp807248154-1262) [n:127.0.0.1:34777__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=tmp_collection&action=DELETE&wt=javabin&version=2} status=0 QTime=222
[junit4] 2> 124347 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 124350 INFO (zkConnectionManagerCallback-688-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 124350 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 124354 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 124355 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44061/solr ready
[junit4] 2> 124355 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for collection to disappear - collection: tmp_collection failOnTimeout:true timeout (sec):15
[junit4] 2> 124357 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.AbstractDistribZkTestBase Collection has disappeared - collection: tmp_collection
[junit4] 2> 124461 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 124465 INFO (zkConnectionManagerCallback-693-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 124465 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 124466 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 124468 INFO (TEST-CdcrReplicationHandlerTest.testFullReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44061/solr ready
[junit4] 2> 124473 INFO (qtp807248154-1263) [n:127.0.0.1:34777__uhb ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&maxShardsPerNode=1&collection.configName=conf1&name=source_collection&action=CREATE&numShards=1&createNodeSet=127.0.0.1:39607__uhb,127.0.0.1:34777__uhb&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 124481 INFO (OverseerThreadFactory-630-thread-5-processing-n:127.0.0.1:39607__uhb) [n:127.0.0.1:39607__uhb ] o.a.s.c.a.c.CreateCollectionCmd Create collection source_collection
[junit4] 2> 124481 INFO (OverseerCollectionConfigSetProcessor-73921418611458052-127.0.0.1:39607__uhb-n_0000000000) [n:127.0.0.1:39607__uhb ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 124593 INFO (qtp807248154-1262) [n:127.0.0.1:34777__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 124594 INFO (qtp1186806583-1228) [n:127.0.0.1:39607__uhb ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 124598 INFO (qtp1186806583-1229) [n:127.0.0.1:39607__uhb x:source_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=source_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=source_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 124600 INFO (qtp807248154-1261) [n:127.0.0.1:34777__uhb x:source_collection_shard1_replica_n2 ] o.a.s.h.a.CoreAdminOperation core crea
[...truncated too long message...]
ST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2f9298b7: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm/com.codahale.metrics.MetricRegistry@6727347c
[junit4] 2> 233306 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty tag=null
[junit4] 2> 233306 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4c13bce9: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty/com.codahale.metrics.MetricRegistry@18793693
[junit4] 2> 233306 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster tag=null
[junit4] 2> 233307 INFO (closeThreadPool-2133-thread-1) [ ] o.a.s.c.Overseer Overseer (id=73921425125015579-127.0.0.1:45115__uhb-n_0000000002) closing
[junit4] 2> 233307 INFO (OverseerStateUpdate-73921425125015579-127.0.0.1:45115__uhb-n_0000000002) [n:127.0.0.1:45115__uhb ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45115__uhb
[junit4] 2> 233310 INFO (OverseerAutoScalingTriggerThread-73921425125015579-127.0.0.1:45115__uhb-n_0000000002) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed, exiting.
[junit4] 2> 233311 INFO (closeThreadPool-2133-thread-2) [ ] o.a.s.c.Overseer Overseer (id=73921425125015579-127.0.0.1:45115__uhb-n_0000000002) closing
[junit4] 2> 233801 WARN (cdcr-update-log-synchronizer-2112-thread-1) [ ] o.a.s.h.CdcrUpdateLogSynchronizer Couldn't get last processed version from leader http://127.0.0.1:37011/_uhb/source_collection_shard1_replica_n1/: Server refused connection at: http://127.0.0.1:37011/_uhb/source_collection_shard1_replica_n1
[junit4] 2> 234802 WARN (cdcr-update-log-synchronizer-2112-thread-1) [ ] o.a.s.h.CdcrUpdateLogSynchronizer Couldn't get last processed version from leader http://127.0.0.1:37011/_uhb/source_collection_shard1_replica_n1/: Server refused connection at: http://127.0.0.1:37011/_uhb/source_collection_shard1_replica_n1
[junit4] 2> 235791 WARN (zkCallback-2081-thread-4) [n:127.0.0.1:45115__uhb c:source_collection s:shard1 r:core_node4 x:source_collection_shard1_replica_n2 ] o.a.s.c.SyncStrategy Closed, skipping sync up.
[junit4] 2> 235791 INFO (zkCallback-2081-thread-4) [n:127.0.0.1:45115__uhb c:source_collection s:shard1 r:core_node4 x:source_collection_shard1_replica_n2 ] o.a.s.c.SolrCore [source_collection_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@608e12a5
[junit4] 2> 235791 INFO (zkCallback-2081-thread-4) [n:127.0.0.1:45115__uhb c:source_collection s:shard1 r:core_node4 x:source_collection_shard1_replica_n2 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.source_collection.shard1.replica_n2 tag=SolrCore@608e12a5
[junit4] 2> 235792 INFO (zkCallback-2081-thread-4) [n:127.0.0.1:45115__uhb c:source_collection s:shard1 r:core_node4 x:source_collection_shard1_replica_n2 ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1463c6c1: rootName = null, domain = solr.core.source_collection.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.source_collection.shard1.replica_n2/com.codahale.metrics.MetricRegistry@3edde6f2
[junit4] 2> 235804 INFO (zkCallback-2081-thread-4) [n:127.0.0.1:45115__uhb c:source_collection s:shard1 r:core_node4 x:source_collection_shard1_replica_n2 ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.source_collection.shard1.leader tag=SolrCore@608e12a5
[junit4] 2> 235804 INFO (zkCallback-2081-thread-4) [n:127.0.0.1:45115__uhb c:source_collection s:shard1 r:core_node4 x:source_collection_shard1_replica_n2 ] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ source_collection:shard1
[junit4] 2> 235804 WARN (cdcr-update-log-synchronizer-2112-thread-1) [ ] o.a.s.h.CdcrUpdateLogSynchronizer Couldn't get last processed version from leader http://127.0.0.1:37011/_uhb/source_collection_shard1_replica_n1/: Server refused connection at: http://127.0.0.1:37011/_uhb/source_collection_shard1_replica_n1
[junit4] 2> 235805 INFO (zkCallback-2081-thread-4) [n:127.0.0.1:45115__uhb c:source_collection s:shard1 r:core_node4 x:source_collection_shard1_replica_n2 ] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter.close() ... SKIPPED (unnecessary).
[junit4] 2> 235910 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.Overseer Overseer (id=73921425125015579-127.0.0.1:45115__uhb-n_0000000002) closing
[junit4] 2> 235911 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@fbaa102{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:45115}
[junit4] 2> 235912 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@69b84550{/_uhb,null,UNAVAILABLE}
[junit4] 2> 235912 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 235912 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testPartialReplication
[junit4] 2> 235912 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
[junit4] 2> 236115 WARN (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 22 /solr/aliases.json
[junit4] 2> 14 /solr/collections/source_collection/terms/shard1
[junit4] 2> 7 /solr/collections/source_collection/leaders/shard1/leader
[junit4] 2> 7 /solr/collections/tmp_collection/state.json
[junit4] 2> 7 /solr/collections/tmp_collection/terms/shard1
[junit4] 2> 5 /solr/configs/conf1
[junit4] 2> 3 /solr/packages.json
[junit4] 2> 3 /solr/security.json
[junit4] 2> 3 /solr/collections/tmp_collection/leaders/shard1/leader
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 47 /solr/collections/source_collection/state.json
[junit4] 2> 24 /solr/collections/tmp_collection/state.json
[junit4] 2> 22 /solr/clusterprops.json
[junit4] 2> 22 /solr/clusterstate.json
[junit4] 2> 4 /solr/collections/source_collection/cdcr/state/buffer
[junit4] 2> 4 /solr/collections/source_collection/cdcr/state/process
[junit4] 2> 2 /solr/collections/tmp_collection/cdcr/state/buffer
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2> 2 /solr/collections/source_collection/leader_elect/shard1/election/73921425125015556-core_node3-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/73921425125015556-127.0.0.1:37011__uhb-n_0000000000
[junit4] 2> 2 /solr/collections/tmp_collection/cdcr/state/process
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 44 /solr/overseer/queue
[junit4] 2> 31 /solr/collections
[junit4] 2> 29 /solr/live_nodes
[junit4] 2> 17 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/autoscaling/events/.scheduled_maintenance
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> 236117 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:34653
[junit4] 2> 236117 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:34653
[junit4] 2> 236117 INFO (TEST-CdcrReplicationHandlerTest.testPartialReplication-seed#[787D1CF68CEFFA2E]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 34653
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrReplicationHandlerTest_787D1CF68CEFFA2E-001
[junit4] 2> Jun 13, 2020 3:17:10 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene86): {_root_=PostingsFormat(name=Asserting), id=FST50}, docValues:{_version_=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=157, maxMBSortInHeap=6.342482384784217, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@30a86f06), locale=it-CH, timezone=Europe/Zaporozhye
[junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=209158832,total=470286336
[junit4] 2> NOTE: All tests run in this JVM: [BasicAuthStandaloneTest, ForceLeaderWithTlogReplicasTest, TokenizerChainTest, ReplicationFactorTest, XsltUpdateRequestHandlerTest, TestCollationFieldDocValues, TestRTGBase, CSVRequestHandlerTest, RequestLoggingTest, ParsingFieldUpdateProcessorsTest, CdcrReplicationHandlerTest]
[junit4] Completed [43/910 (1!)] on J2 in 118.16s, 5 tests, 1 failure <<< FAILURES!
[...truncated 54973 lines...]
[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 475 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/475/
2 tests failed.
FAILED: org.apache.lucene.util.TestOfflineSorter.testThreadSafety
Error Message:
Captured an uncaught exception in thread: Thread[id=27569, name=Thread-26836, state=RUNNABLE, group=TGRP-TestOfflineSorter]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=27569, name=Thread-26836, state=RUNNABLE, group=TGRP-TestOfflineSorter]
Caused by: java.lang.RuntimeException: java.nio.file.AccessDeniedException: Can't open a file still open for writing: unsorted_tmp_0.tmp
at __randomizedtesting.SeedInfo.seed([C5A1C5B4802C63C9]:0)
at org.apache.lucene.util.TestOfflineSorter$2.run(TestOfflineSorter.java:272)
Caused by: java.nio.file.AccessDeniedException: Can't open a file still open for writing: unsorted_tmp_0.tmp
at org.apache.lucene.store.ByteBuffersDirectory$FileEntry.openInput(ByteBuffersDirectory.java:251)
at org.apache.lucene.store.ByteBuffersDirectory.openInput(ByteBuffersDirectory.java:218)
at org.apache.lucene.store.FileSwitchDirectory.openInput(FileSwitchDirectory.java:233)
at org.apache.lucene.store.NRTCachingDirectory.slowFileExists(NRTCachingDirectory.java:280)
at org.apache.lucene.store.NRTCachingDirectory.createTempOutput(NRTCachingDirectory.java:254)
at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:707)
at org.apache.lucene.util.TestOfflineSorter.checkSort(TestOfflineSorter.java:184)
at org.apache.lucene.util.TestOfflineSorter.access$100(TestOfflineSorter.java:48)
at org.apache.lucene.util.TestOfflineSorter$2.run(TestOfflineSorter.java:268)
FAILED: org.apache.solr.cloud.ForceLeaderTest.testReplicasInLowerTerms
Error Message:
Timeout occurred while waiting response from server at: http://127.0.0.1:37612/forceleader_lower_terms_collection
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:37612/forceleader_lower_terms_collection
at __randomizedtesting.SeedInfo.seed([EFBCD81D4509DB55:6188F85E85AE7B21]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
at org.apache.solr.cloud.HttpPartitionTest.realTimeGetDocId(HttpPartitionTest.java:578)
at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:563)
at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:508)
at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLowerTerms(ForceLeaderTest.java:151)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method)
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
at java.net.SocketInputStream.read(SocketInputStream.java:171)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
... 48 more
Build Log:
[...truncated 2150 lines...]
[junit4] Suite: org.apache.lucene.util.TestOfflineSorter
[junit4] 2> juin 16, 2020 12:36:39 AM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
[junit4] 2> AVERTISSEMENT: Uncaught exception in thread: Thread[Thread-26836,5,TGRP-TestOfflineSorter]
[junit4] 2> java.lang.RuntimeException: java.nio.file.AccessDeniedException: Can't open a file still open for writing: unsorted_tmp_0.tmp
[junit4] 2> at __randomizedtesting.SeedInfo.seed([C5A1C5B4802C63C9]:0)
[junit4] 2> at org.apache.lucene.util.TestOfflineSorter$2.run(TestOfflineSorter.java:272)
[junit4] 2> Caused by: java.nio.file.AccessDeniedException: Can't open a file still open for writing: unsorted_tmp_0.tmp
[junit4] 2> at org.apache.lucene.store.ByteBuffersDirectory$FileEntry.openInput(ByteBuffersDirectory.java:251)
[junit4] 2> at org.apache.lucene.store.ByteBuffersDirectory.openInput(ByteBuffersDirectory.java:218)
[junit4] 2> at org.apache.lucene.store.FileSwitchDirectory.openInput(FileSwitchDirectory.java:233)
[junit4] 2> at org.apache.lucene.store.NRTCachingDirectory.slowFileExists(NRTCachingDirectory.java:280)
[junit4] 2> at org.apache.lucene.store.NRTCachingDirectory.createTempOutput(NRTCachingDirectory.java:254)
[junit4] 2> at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:707)
[junit4] 2> at org.apache.lucene.util.TestOfflineSorter.checkSort(TestOfflineSorter.java:184)
[junit4] 2> at org.apache.lucene.util.TestOfflineSorter.access$100(TestOfflineSorter.java:48)
[junit4] 2> at org.apache.lucene.util.TestOfflineSorter$2.run(TestOfflineSorter.java:268)
[junit4] 2>
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestOfflineSorter -Dtests.method=testThreadSafety -Dtests.seed=C5A1C5B4802C63C9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=fr-BE -Dtests.timezone=Africa/Monrovia -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
[junit4] ERROR 0.07s J1 | TestOfflineSorter.testThreadSafety <<<
[junit4] > Throwable #1: java.lang.AssertionError
[junit4] > at org.apache.lucene.util.TestOfflineSorter.testThreadSafety(TestOfflineSorter.java:283)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=27569, name=Thread-26836, state=RUNNABLE, group=TGRP-TestOfflineSorter]
[junit4] > Caused by: java.lang.RuntimeException: java.nio.file.AccessDeniedException: Can't open a file still open for writing: unsorted_tmp_0.tmp
[junit4] > at __randomizedtesting.SeedInfo.seed([C5A1C5B4802C63C9]:0)
[junit4] > at org.apache.lucene.util.TestOfflineSorter$2.run(TestOfflineSorter.java:272)
[junit4] > Caused by: java.nio.file.AccessDeniedException: Can't open a file still open for writing: unsorted_tmp_0.tmp
[junit4] > at org.apache.lucene.store.ByteBuffersDirectory$FileEntry.openInput(ByteBuffersDirectory.java:251)
[junit4] > at org.apache.lucene.store.ByteBuffersDirectory.openInput(ByteBuffersDirectory.java:218)
[junit4] > at org.apache.lucene.store.FileSwitchDirectory.openInput(FileSwitchDirectory.java:233)
[junit4] > at org.apache.lucene.store.NRTCachingDirectory.slowFileExists(NRTCachingDirectory.java:280)
[junit4] > at org.apache.lucene.store.NRTCachingDirectory.createTempOutput(NRTCachingDirectory.java:254)
[junit4] > at org.apache.lucene.store.MockDirectoryWrapper.createTempOutput(MockDirectoryWrapper.java:707)
[junit4] > at org.apache.lucene.util.TestOfflineSorter.checkSort(TestOfflineSorter.java:184)
[junit4] > at org.apache.lucene.util.TestOfflineSorter.access$100(TestOfflineSorter.java:48)
[junit4] > at org.apache.lucene.util.TestOfflineSorter$2.run(TestOfflineSorter.java:268)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/lucene/build/core/test/J1/temp/lucene.util.TestOfflineSorter_C5A1C5B4802C63C9-001
[junit4] 2> NOTE: test params are: codec=FastDecompressionCompressingStoredFieldsData(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=1, maxDocsPerChunk=8, blockShift=2), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=1, blockSize=2)), sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@5653347d), locale=fr-BE, timezone=Africa/Monrovia
[junit4] 2> NOTE: Linux 4.4.0-170-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=8,threads=1,free=262430024,total=477626368
[junit4] 2> NOTE: All tests run in this JVM: [TestDocument, TestCustomSearcherSort, TestTotalHitCountCollector, TestTieredMergePolicy, TestBooleanQueryVisitSubscorers, TestDocValuesFieldUpdates, TestAllFilesCheckIndexHeader, MultiCollectorTest, TestDocsWithFieldSet, TestPolygon2D, TestIndependenceStandardized, TestSearch, TestTermScorer, TestNativeFSLockFactory, TestFileSwitchDirectory, TestLatLonPointDistanceFeatureQuery, TestIOUtils, TestSpanMultiTermQueryWrapper, TestPositionIncrement, TestIndexWriterWithThreads, TestReaderClosed, TestIndexedDISI, TestCodecUtil, TestXYShapeEncoding, TestSegmentTermDocs, Test2BPostingsBytes, TestLowercaseAsciiCompression, TestMatchesIterator, TestNeedsScores, TestTopDocsMerge, TestLucene50LiveDocsFormat, TestDocValuesRewriteMethod, TestMinimize, TestAutomatonQuery, TestCachingTokenFilter, TestBKDRadixSelector, TestLucene80NormsFormatMergeInstance, TestWeakIdentityMap, TestInfoStream, TestBytesStore, TestPerSegmentDeletes, TestBasicModelIF, TestLongValuesSource, TestStopFilter, TestParallelReaderEmptyIndex, TestSumDocFreq, TestNamedSPILoader, TestNoDeletionPolicy, TestXYDocValuesQueries, TestDeletionPolicy, TestCrashCausesCorruptIndex, TestPoint, TestWANDScorer, TestSimilarity2, TestBagOfPositions, TestDocIDMerger, TestFieldsReader, TestGeoEncodingUtils, TestSetOnce, TestMutablePointsReaderUtils, TestConstantScoreQuery, TestBooleanSimilarity, TestOrdinalMap, TestBytesRef, TestStressIndexing, TestPayloadsOnVectors, TestMultiTermsEnum, TestArrayUtil, TestPostingsOffsets, TestConcurrentMergeScheduler, TestIndexWriterReader, TestSortedSetDocValues, TestFuzzyQuery, TestFilterLeafReader, TestMSBRadixSorter, TestLatLonDocValuesField, TestGraphTokenFilter, TestSpans, TestSpanExplanations, TestDocumentsWriterStallControl, TestFixedBitDocIdSet, TestNRTThreads, TestAxiomaticSimilarity, TestSloppyPhraseQuery, TestIndexWriter, TestBooleanMinShouldMatch, TestLazyProxSkipping, TestMultiThreadTermVectors, TestCharsRef, TestRollback, TestFieldValueQuery, TestLatLonPolygonShapeQueries, TestDemo, TestFrozenBufferedUpdates, TestAnalyzerWrapper, TestCharArraySet, TestCharFilter, TestCharacterUtils, TestDelegatingAnalyzerWrapper, TestReusableStringReader, TestWordlistLoader, TestStandardAnalyzer, TestBytesRefAttImpl, TestPackedTokenAttributeImpl, TestSimpleAttributeImpl, TestFastDecompressionMode, TestHighCompressionMode, TestLucene50CompoundFormat, TestLucene50StoredFieldsFormatMergeInstance, TestLucene50TermVectorsFormat, TestLucene84PostingsFormat, TestPForUtil, TestLucene86PointsFormat, TestLucene86SegmentInfoFormat, TestPerFieldDocValuesFormat, TestPerFieldPostingsFormat2, TestBinaryDocument, TestDateTools, TestFeatureField, TestFeatureSort, TestField, TestFieldType, TestFloatRange, TestIntRange, TestLatLonMultiPointShapeQueries, TestLatLonMultiPolygonShapeQueries, TestLatLonPoint, TestLongDistanceFeatureQuery, TestLongRange, TestXYLineShapeQueries, TestXYMultiLineShapeQueries, TestXYShape, TestCircle2D, TestLine2D, TestPolygon, TestTessellator, TestXYCircle, TestXYLine, TestXYPoint, TestXYPolygon, Test2BBinaryDocValues, Test2BDocs, Test2BNumericDocValues, Test2BPoints, Test2BPositions, Test2BTerms, Test4GBStoredFields, TestAllFilesHaveChecksumFooter, TestAllFilesHaveCodecHeader, TestBinaryTerms, TestBufferedUpdates, TestByteSlices, TestCheckIndex, TestCodecHoldsOpenFiles, TestCrash, TestCustomTermFreq, TestDirectoryReader, TestDirectoryReaderReopen, TestDoc, TestDocCount, TestDocValues, TestDocsAndPositions, TestExceedMaxTermLength, TestExitableDirectoryReader, TestFieldInfos, TestFieldInvertState, TestFieldReuse, TestFilterDirectoryReader, TestFlushByRamOrCountsPolicy, TestForTooMuchCloning, TestIndexInput, TestIndexManyDocuments, TestIndexReaderClose, TestIndexSorting, TestIndexWriterDelete, TestIndexWriterDeleteByQuery, TestIndexWriterExceptions2, TestIndexWriterForceMerge, TestIndexWriterFromReader, TestIndexWriterLockRelease, TestIndexWriterMaxDocs, TestIndexWriterMerging, TestIndexWriterNRTIsCurrent, TestIndexWriterOnDiskFull, TestIndexWriterOnJRECrash, TestIndexWriterOnVMError, TestIsCurrent, TestManyFields, TestMaxPosition, TestMergeRateLimiter, TestMixedCodecs, TestMultiDocValues, TestMultiFields, TestNorms, TestNumericDocValuesUpdates, TestTermStates, TestTermVectorsReader, TestTermdocPerf, TestTerms, TestTermsEnum2, TestTransactionRollback, TestTransactions, TestTryDelete, TestUniqueTermCount, TestUpgradeIndexMergePolicy, TestApproximationSearchEquivalence, TestBlockMaxConjunction, TestBoolean2, TestBoolean2ScorerSupplier, TestBooleanScorer, TestBoostQuery, TestCachingCollector, TestComplexExplanationsOfNonMatches, TestConjunctionDISI, TestConjunctions, TestDisjunctionMaxQuery, TestDisjunctionScoreBlockBoundaryPropagator, TestDocIdSetIterator, TestEarlyTermination, TestElevationComparator, TestFloatRangeFieldQueries, TestIndexSearcher, TestIntRangeFieldQueries, TestLatLonPointQueries, TestPrefixInBooleanQuery, TestPrefixQuery, TestPrefixRandom, TestQueryVisitor, TestRangeFieldsDocValuesQuery, TestRegexpRandom, TestRegexpRandom2, TestReqExclBulkScorer, TestReqOptSumScorer, TestSameScoresWithThreads, TestScorerPerf, TestSearcherManager, TestAxiomaticF1EXP, TestAxiomaticF1LOG, TestAxiomaticF2LOG, TestAxiomaticF3EXP, TestAxiomaticF3LOG, TestBasicModelG, TestDistributionLL, TestDistributionSPL, TestIndependenceChiSquared, TestIndependenceSaturated, TestLMJelinekMercerSimilarity, TestBasics, TestFieldMaskingSpanQuery, TestFilterSpans, TestNearSpansOrdered, TestSpanBoostQuery, TestSpanCollection, TestSpanFirstQuery, TestSpanNotQuery, TestSpanOrQuery, TestSpanSearchEquivalence, TestBufferedChecksum, TestBufferedIndexInput, TestByteArrayDataInput, TestByteBuffersDirectory, TestGrowableByteArrayDataOutput, TestHugeRamFile, TestLockFactory, TestMultiMMap, TestNIOFSDirectory, TestSimpleFSDirectory, TestSimpleFSLockFactory, TestSleepingLockWrapper, TestTrackingDirectoryWrapper, Test2BPagedBytes, TestBytesRefHash, TestCollectionUtil, TestDocIdSetBuilder, TestFixedLengthBytesRefArray, TestFrequencyTrackingRingBuffer, TestFutureArrays, TestFutureObjects, TestIntArrayDocIdSet, TestIntroSelector, TestIntroSorter, TestIntsRef, TestLSBRadixSorter, TestLongBitSet, TestLongsRef, TestMergedIterator, TestOfflineSorter]
[junit4] Completed [490/565 (1!)] on J1 in 29.29s, 15 tests, 1 error <<< FAILURES!
[...truncated 12685 lines...]
[junit4] Suite: org.apache.solr.cloud.ForceLeaderTest
[junit4] 2> 3645635 INFO (SUITE-ForceLeaderTest-seed#[EFBCD81D4509DB55]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/server/solr/configsets/_default/conf'
[junit4] 2> 3645635 INFO (SUITE-ForceLeaderTest-seed#[EFBCD81D4509DB55]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3645636 INFO (SUITE-ForceLeaderTest-seed#[EFBCD81D4509DB55]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/data-dir-179-001
[junit4] 2> 3645636 WARN (SUITE-ForceLeaderTest-seed#[EFBCD81D4509DB55]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=32 numCloses=32
[junit4] 2> 3645636 INFO (SUITE-ForceLeaderTest-seed#[EFBCD81D4509DB55]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3645638 INFO (SUITE-ForceLeaderTest-seed#[EFBCD81D4509DB55]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 3645638 INFO (SUITE-ForceLeaderTest-seed#[EFBCD81D4509DB55]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 3645645 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3645646 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
[junit4] 2> 3645646 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3645746 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer start zk server on port: 39243
[junit4] 2> 3645746 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:39243
[junit4] 2> 3645746 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:39243
[junit4] 2> 3645746 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 39243
[junit4] 2> 3645747 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3645753 INFO (zkConnectionManagerCallback-19214-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3645753 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3645757 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3645758 INFO (zkConnectionManagerCallback-19216-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3645758 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3645760 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 3645762 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
[junit4] 2> 3645764 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 3645766 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 3645767 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 3645769 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 3645770 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 3645771 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 3645773 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 3645774 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 3645775 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 3645777 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 3645890 WARN (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 3645891 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 3645891 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 3645891 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 3645892 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3645892 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3645892 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3645892 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@33a48e8{/,null,AVAILABLE}
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@62414773{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:46061}
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.s.Server Started @3645927ms
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/tempDir-001/control/data, hostContext=/, hostPort=44897, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/control-001/cores}
[junit4] 2> 3645893 ERROR (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 3645893 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-16T04:07:24.769Z
[junit4] 2> 3645894 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3645895 INFO (zkConnectionManagerCallback-19218-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3645895 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3645997 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3645997 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/control-001/solr.xml
[junit4] 2> 3646002 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3646002 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3646004 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3647000 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 3647001 WARN (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@22a9e145[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3647001 WARN (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@22a9e145[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3647004 WARN (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2e245130[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3647004 WARN (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2e245130[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3647005 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39243/solr
[junit4] 2> 3647006 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3647007 INFO (zkConnectionManagerCallback-19229-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3647007 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3647110 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3647111 INFO (zkConnectionManagerCallback-19231-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3647111 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3647211 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44897_
[junit4] 2> 3647212 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.c.Overseer Overseer (id=72978766750285828-127.0.0.1:44897_-n_0000000000) starting
[junit4] 2> 3647219 INFO (OverseerStateUpdate-72978766750285828-127.0.0.1:44897_-n_0000000000) [n:127.0.0.1:44897_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:44897_
[junit4] 2> 3647219 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44897_
[junit4] 2> 3647222 INFO (OverseerStateUpdate-72978766750285828-127.0.0.1:44897_-n_0000000000) [n:127.0.0.1:44897_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3647225 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 3647225 WARN (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 3647247 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3647287 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3647302 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3647302 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3647304 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [n:127.0.0.1:44897_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/control-001/cores
[junit4] 2> 3647325 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3647326 INFO (zkConnectionManagerCallback-19248-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3647326 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3647328 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3647329 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39243/solr ready
[junit4] 2> 3647330 INFO (SocketProxy-Acceptor-44897) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=32888,localport=44897], receiveBufferSize: 531000
[junit4] 2> 3647330 INFO (SocketProxy-Acceptor-44897) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46061,localport=56710], receiveBufferSize=530904
[junit4] 2> 3647331 INFO (qtp1163998517-37487) [n:127.0.0.1:44897_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44897_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3647337 INFO (OverseerThreadFactory-19238-thread-1-processing-n:127.0.0.1:44897_) [n:127.0.0.1:44897_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 3647443 INFO (SocketProxy-Acceptor-44897) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=32900,localport=44897], receiveBufferSize: 531000
[junit4] 2> 3647444 INFO (SocketProxy-Acceptor-44897) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46061,localport=56722], receiveBufferSize=530904
[junit4] 2> 3647445 INFO (qtp1163998517-37486) [n:127.0.0.1:44897_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3647445 INFO (qtp1163998517-37486) [n:127.0.0.1:44897_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 3647451 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3648468 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 3648500 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=test
[junit4] 2> 3648644 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3648665 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from configset conf1, trusted=true
[junit4] 2> 3648665 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3648666 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 3648669 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4fd905a7
[junit4] 2> 3648671 WARN (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 3648734 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3648734 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3648743 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3648743 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3648745 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3359094221371901]
[junit4] 2> 3648746 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 3648747 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 3648747 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
[junit4] 2> 3648747 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1669627062646734848
[junit4] 2> 3648752 INFO (searcherExecutor-19250-thread-1-processing-n:127.0.0.1:44897_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher autowarm time: 0 ms
[junit4] 2> 3648754 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 3648754 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 3648759 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3648759 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3648759 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:44897/control_collection_shard1_replica_n1/
[junit4] 2> 3648759 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3648760 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:44897/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 3648760 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/72978766750285828-core_node2-n_0000000000
[junit4] 2> 3648762 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:44897/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 3648864 INFO (zkCallback-19230-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3648864 INFO (zkCallback-19230-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3648865 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3648867 INFO (qtp1163998517-37488) [n:127.0.0.1:44897_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1416
[junit4] 2> 3648871 INFO (qtp1163998517-37487) [n:127.0.0.1:44897_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 3648968 INFO (zkCallback-19230-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3648968 INFO (zkCallback-19230-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3648968 INFO (zkCallback-19230-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3648968 INFO (qtp1163998517-37487) [n:127.0.0.1:44897_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:44897_&wt=javabin&version=2} status=0 QTime=1637
[junit4] 2> 3648969 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
[junit4] 2> 3649074 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3649075 INFO (zkConnectionManagerCallback-19259-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3649075 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3649077 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3649078 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39243/solr ready
[junit4] 2> 3649078 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 3649079 INFO (SocketProxy-Acceptor-44897) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=32936,localport=44897], receiveBufferSize: 531000
[junit4] 2> 3649079 INFO (SocketProxy-Acceptor-44897) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46061,localport=56758], receiveBufferSize=530904
[junit4] 2> 3649080 INFO (qtp1163998517-37486) [n:127.0.0.1:44897_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3649083 INFO (OverseerThreadFactory-19238-thread-2-processing-n:127.0.0.1:44897_) [n:127.0.0.1:44897_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 3649083 INFO (OverseerCollectionConfigSetProcessor-72978766750285828-127.0.0.1:44897_-n_0000000000) [n:127.0.0.1:44897_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3649297 WARN (OverseerThreadFactory-19238-thread-2-processing-n:127.0.0.1:44897_) [n:127.0.0.1:44897_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 3649299 INFO (qtp1163998517-37486) [n:127.0.0.1:44897_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 3649300 INFO (qtp1163998517-37486) [n:127.0.0.1:44897_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=2&wt=javabin&version=2} status=0 QTime=220
[junit4] 2> 3649301 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected: 2
[junit4] 2> 3649301 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
[junit4] 2> 3649301 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected: 2
[junit4] 2> 3649301 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
[junit4] 2> 3649302 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected: 2
[junit4] 2> 3649302 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
[junit4] 2> 3649302 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=3
[junit4] 2> 3649419 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-1-001 of type NRT for shard2
[junit4] 2> 3649421 WARN (closeThreadPool-19260-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 3649421 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 3649421 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 3649421 INFO (closeThreadPool-19260-thread-1) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 3649422 INFO (closeThreadPool-19260-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3649422 INFO (closeThreadPool-19260-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3649422 INFO (closeThreadPool-19260-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3649422 INFO (closeThreadPool-19260-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@35a56cbc{/,null,AVAILABLE}
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7d370141{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:46346}
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.e.j.s.Server Started @3649458ms
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=37612, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-1-001/cores}
[junit4] 2> 3649423 ERROR (closeThreadPool-19260-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 3649423 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-16T04:07:28.299Z
[junit4] 2> 3649424 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3649425 INFO (zkConnectionManagerCallback-19262-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3649425 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3649527 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3649527 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-1-001/solr.xml
[junit4] 2> 3649532 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3649532 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3649533 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3649557 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-2-001 of type NRT for shard1
[junit4] 2> 3649558 WARN (closeThreadPool-19260-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 3649558 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 3649558 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 3649558 INFO (closeThreadPool-19260-thread-2) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 3649560 INFO (closeThreadPool-19260-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3649560 INFO (closeThreadPool-19260-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3649560 INFO (closeThreadPool-19260-thread-2) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 3649560 INFO (closeThreadPool-19260-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3d6d4b88{/,null,AVAILABLE}
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@7b851a59{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:33343}
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.e.j.s.Server Started @3649596ms
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=34220, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-2-001/cores}
[junit4] 2> 3649561 ERROR (closeThreadPool-19260-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 3649561 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-16T04:07:28.437Z
[junit4] 2> 3649562 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3649563 INFO (zkConnectionManagerCallback-19267-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3649563 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3649665 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3649665 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-2-001/solr.xml
[junit4] 2> 3649677 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3649677 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3649678 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3649726 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-3-001 of type NRT for shard2
[junit4] 2> 3649727 WARN (closeThreadPool-19260-thread-3) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 3649727 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 3649727 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 3649727 INFO (closeThreadPool-19260-thread-3) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 3649728 INFO (closeThreadPool-19260-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3649728 INFO (closeThreadPool-19260-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3649729 INFO (closeThreadPool-19260-thread-3) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 3649729 INFO (closeThreadPool-19260-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@62fbcb64{/,null,AVAILABLE}
[junit4] 2> 3649729 INFO (closeThreadPool-19260-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4e0e8ce5{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:36781}
[junit4] 2> 3649729 INFO (closeThreadPool-19260-thread-3) [ ] o.e.j.s.Server Started @3649764ms
[junit4] 2> 3649729 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/tempDir-001/jetty3, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/, hostPort=41690, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-3-001/cores}
[junit4] 2> 3649730 ERROR (closeThreadPool-19260-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3649730 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 3649730 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 3649730 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3649730 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 3649730 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-16T04:07:28.606Z
[junit4] 2> 3649731 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3649732 INFO (zkConnectionManagerCallback-19272-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3649732 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3649833 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 3649833 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-3-001/solr.xml
[junit4] 2> 3649849 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 3649849 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 3649851 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3650455 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 3650456 WARN (closeThreadPool-19260-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@72aa098c[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650456 WARN (closeThreadPool-19260-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@72aa098c[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650463 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 3650463 WARN (closeThreadPool-19260-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@764bfcc1[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650463 WARN (closeThreadPool-19260-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@764bfcc1[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650467 WARN (closeThreadPool-19260-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@10bdded4[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650467 WARN (closeThreadPool-19260-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@10bdded4[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650467 WARN (closeThreadPool-19260-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1c5c752d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650467 WARN (closeThreadPool-19260-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1c5c752d[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3650468 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39243/solr
[junit4] 2> 3650468 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39243/solr
[junit4] 2> 3650469 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3650469 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3650470 INFO (zkConnectionManagerCallback-19290-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3650470 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3650470 INFO (zkConnectionManagerCallback-19291-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3650471 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3650573 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3650574 INFO (zkConnectionManagerCallback-19293-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3650574 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3650581 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3650587 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.ZkController Publish node=127.0.0.1:41690_ as DOWN
[junit4] 2> 3650588 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3650588 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41690_
[junit4] 2> 3650589 INFO (zkCallback-19230-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3650589 INFO (zkCallback-19292-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3650591 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 3650591 WARN (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 3650595 INFO (zkCallback-19258-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3650611 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3650653 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3650680 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3650680 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3650680 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3650682 INFO (closeThreadPool-19260-thread-3) [n:127.0.0.1:41690_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-3-001/cores
[junit4] 2> 3650688 INFO (zkConnectionManagerCallback-19302-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3650688 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3650694 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3650700 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.ZkController Publish node=127.0.0.1:34220_ as DOWN
[junit4] 2> 3650701 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3650701 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34220_
[junit4] 2> 3650702 INFO (zkCallback-19230-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3650702 INFO (zkCallback-19258-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3650702 INFO (zkCallback-19292-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3650702 INFO (zkCallback-19301-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3650706 INFO (closeThreadPool-19260-thread-3) [ ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:41690_
[junit4] 2> 3650706 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 3650707 WARN (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 3650726 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3650773 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3650792 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3650792 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3650794 INFO (closeThreadPool-19260-thread-2) [n:127.0.0.1:34220_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-2-001/cores
[junit4] 2> 3650835 INFO (closeThreadPool-19260-thread-2) [ ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:34220_
[junit4] 2> 3651094 INFO (OverseerCollectionConfigSetProcessor-72978766750285828-127.0.0.1:44897_-n_0000000000) [n:127.0.0.1:44897_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3652326 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 3652327 WARN (closeThreadPool-19260-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7e2bc868[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3652327 WARN (closeThreadPool-19260-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7e2bc868[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3652329 WARN (closeThreadPool-19260-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@fa04d13[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3652329 WARN (closeThreadPool-19260-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@fa04d13[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 3652331 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39243/solr
[junit4] 2> 3652331 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3652332 INFO (zkConnectionManagerCallback-19319-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3652333 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3652435 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 3652436 INFO (zkConnectionManagerCallback-19321-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3652436 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 3652441 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3652446 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.ZkController Publish node=127.0.0.1:37612_ as DOWN
[junit4] 2> 3652447 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 3652447 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37612_
[junit4] 2> 3652449 INFO (zkCallback-19292-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3652449 INFO (zkCallback-19258-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3652449 INFO (zkCallback-19230-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3652449 INFO (zkCallback-19320-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3652449 INFO (zkCallback-19301-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3652460 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 3652460 WARN (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 3652504 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 3652572 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3652598 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3652598 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7c664033
[junit4] 2> 3652613 INFO (closeThreadPool-19260-thread-1) [n:127.0.0.1:37612_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001/shard-1-001/cores
[junit4] 2> 3652657 INFO (closeThreadPool-19260-thread-1) [ ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:37612_
[junit4] 2> 3652660 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=60666,localport=34220], receiveBufferSize: 531000
[junit4] 2> 3652661 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33343,localport=50922], receiveBufferSize=530904
[junit4] 2> 3652661 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=60664,localport=34220], receiveBufferSize: 531000
[junit4] 2> 3652661 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33343,localport=50926], receiveBufferSize=530904
[junit4] 2> 3652661 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=60670,localport=34220], receiveBufferSize: 531000
[junit4] 2> 3652662 INFO (qtp2142933366-37563) [n:127.0.0.1:34220_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:41690_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3652662 INFO (qtp2142933366-37565) [n:127.0.0.1:34220_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:34220_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3652662 INFO (qtp2142933366-37564) [n:127.0.0.1:34220_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37612_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3652663 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33343,localport=50928], receiveBufferSize=530904
[junit4] 2> 3652669 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=60684,localport=34220], receiveBufferSize: 531000
[junit4] 2> 3652669 INFO (SocketProxy-Acceptor-34220) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=33343,localport=50942], receiveBufferSize=530904
[junit4] 2> 3652670 INFO (qtp2142933366-37652) [n:127.0.0.1:34220_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 3652671 INFO (SocketProxy-Acceptor-37612) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=56454,localport=37612], receiveBufferSize: 531000
[junit4] 2> 3652672 INFO (SocketProxy-Acceptor-37612) [ ] o.a.s.c.s.c.SocketProxy proxy connection Socket[addr=/127.0.0.1,port=46346,localport=38082], receiveBufferSize=530904
[junit4] 2> 3652673 INFO (qtp2030602936-37551) [n:127.0.0.1:37612_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 3652673 INFO (SocketProxy-Acceptor-41690) [ ] o.a.s.c.s.c.SocketProxy accepted Socket[addr=/127.0.0.1,port=36506,localport=41690], receiveBufferSize: 531000
[junit4] 2> 3652674 INFO (SocketProxy-Acceptor
[...truncated too long message...]
unit4] 2> 19 /solr/overseer/collection-queue-work
[junit4] 2> 16 /solr/collections
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2>
[junit4] 2> 3806483 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer waitForServerDown: 127.0.0.1:39243
[junit4] 2> 3806483 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:39243
[junit4] 2> 3806483 INFO (TEST-ForceLeaderTest.testReplicasInLowerTerms-seed#[EFBCD81D4509DB55]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 39243
[junit4] 2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ForceLeaderTest -Dtests.method=testReplicasInLowerTerms -Dtests.seed=EFBCD81D4509DB55 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/test-data/enwiki.random.lines.txt -Dtests.locale=en -Dtests.timezone=Africa/Khartoum -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 161s J1 | ForceLeaderTest.testReplicasInLowerTerms <<<
[junit4] > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occurred while waiting response from server at: http://127.0.0.1:37612/forceleader_lower_terms_collection
[junit4] > at __randomizedtesting.SeedInfo.seed([EFBCD81D4509DB55:6188F85E85AE7B21]:0)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:692)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
[junit4] > at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1290)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.realTimeGetDocId(HttpPartitionTest.java:578)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.assertDocExists(HttpPartitionTest.java:563)
[junit4] > at org.apache.solr.cloud.HttpPartitionTest.assertDocsExistInAllReplicas(HttpPartitionTest.java:508)
[junit4] > at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLowerTerms(ForceLeaderTest.java:151)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > Caused by: java.net.SocketTimeoutException: Read timed out
[junit4] > at java.net.SocketInputStream.socketRead0(Native Method)
[junit4] > at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
[junit4] > at java.net.SocketInputStream.read(SocketInputStream.java:171)
[junit4] > at java.net.SocketInputStream.read(SocketInputStream.java:141)
[junit4] > at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
[junit4] > at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
[junit4] > at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:280)
[junit4] > at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
[junit4] > at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
[junit4] > at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
[junit4] > at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
[junit4] > at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:157)
[junit4] > at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
[junit4] > at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
[junit4] > at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
[junit4] > at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:186)
[junit4] > at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
[junit4] > at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
[junit4] > at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
[junit4] > at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
[junit4] > at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
[junit4] > at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:571)
[junit4] > ... 48 more
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ForceLeaderTest_EFBCD81D4509DB55-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene86): {multiDefault=PostingsFormat(name=Direct), a_t=PostingsFormat(name=Direct), _root_=Lucene84, id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), text=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Lucene80), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=813, maxMBSortInHeap=7.370499341002009, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@5e747bf0), locale=en, timezone=Africa/Khartoum
[junit4] 2> NOTE: Linux 4.4.0-170-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=8,threads=1,free=129881400,total=522190848
[junit4] 2> NOTE: All tests run in this JVM: [TestNumericTerms64, CloudReplicaSourceTest, ExitableDirectoryReaderTest, TestSimDistributedQueue, TestRestoreCore, TestDeprecatedFilters, TestBulkSchemaConcurrent, SimplePostToolTest, CurrencyRangeFacetCloudTest, TestCloudConsistency, OverseerCollectionConfigSetProcessorTest, LargeFieldTest, CircularListTest, TestDocTermOrdsUninvertLimit, OpenExchangeRatesOrgProviderTest, TestReloadDeadlock, MetricTriggerTest, TestMinMaxOnMultiValuedField, TestRawResponseWriter, TestLeaderElectionWithEmptyReplica, DirectUpdateHandlerTest, TestShardHandlerFactory, SolrSlf4jReporterTest, SpatialHeatmapFacetsTest, BigEndianAscendingWordDeserializerTest, ResponseHeaderTest, ZookeeperReadAPITest, TestRecovery, TestCollationField, TestCursorMarkWithoutUniqueKey, TestJsonRequest, PreAnalyzedFieldTest, NumberUtilsTest, TestExceedMaxTermLength, TestRequestForwarding, BasicFunctionalityTest, ShardsWhitelistTest, DistributedUpdateProcessorTest, PingRequestHandlerTest, CollectionTooManyReplicasTest, TriLevelCompositeIdRoutingTest, TestObjectReleaseTracker, SoftAutoCommitTest, SolrGraphiteReporterTest, ZkDynamicConfigTest, TestSimpleQParserPlugin, TestBlendedInfixSuggestions, SpatialRPTFieldTypeTest, DeleteInactiveReplicaTest, ChangedSchemaMergeTest, SuggesterWFSTTest, SpellingQueryConverterTest, TestSort, TestSolrTestCaseJ4, TestConfigSetsAPIZkFailure, HdfsChaosMonkeySafeLeaderTest, SharedFSAutoReplicaFailoverTest, TestSolrCLIRunExample, TestDFRSimilarityFactory, CloudExitableDirectoryReaderTest, TestMacroExpander, TestSchemaVersionResource, SolrMetricManagerTest, TestCollapseQParserPlugin, CdcrVersionReplicationTest, ImplicitSnitchTest, HdfsDirectoryTest, TestDistribDocBasedVersion, HdfsRecoveryZkTest, IndexSizeTriggerSizeEstimationTest, MoreLikeThisComponentTest, TestTrackingShardHandlerFactory, PeerSyncReplicationTest, TestRequestStatusCollectionAPI, TermVectorComponentTest, SignificantTermsQParserPluginTest, TestCloudPhrasesIdentificationComponent, ConcurrentCreateCollectionTest, ShowFileRequestHandlerTest, TestStreamBody, TestLRUCache, TestLazyCores, TestExpandComponent, UninvertDocValuesMergePolicyTest, TestSolr4Spatial2, DistributedFacetExistsSmallTest, TestOmitPositions, TestBM25SimilarityFactory, TestTestInjection, TestHighlightDedupGrouping, SaslZkACLProviderTest, HighlighterTest, HdfsWriteToMultipleCollectionsTest, TestHdfsCloudBackupRestore, TestStressCloudBlindAtomicUpdates, TestFieldCacheSort, ChaosMonkeySafeLeaderWithPullReplicasTest, ReindexCollectionTest, DistributedTermsComponentTest, ClusterStateUpdateTest, StatsReloadRaceTest, BlockJoinFacetRandomTest, TestSimClusterStateProvider, DistributedQueryElevationComponentTest, TestCloudRecovery, NodeMutatorTest, SolrIndexMetricsTest, DistributedQueueTest, LeaderFailoverAfterPartitionTest, ZkNodePropsTest, TestSolrFieldCacheBean, DocValuesMultiTest, TestSystemCollAutoCreate, ResponseBuilderTest, ExternalFileFieldSortTest, SolrCoreTest, TestCustomDocTransformer, TestDynamicFieldNamesIndexCorrectly, MultiDestinationAuditLoggerTest, DaemonStreamApiTest, TestWordDelimiterFilterFactory, TestSimGenericDistributedQueue, SolrInfoBeanTest, SmileWriterTest, ExplicitHLLTest, TestRandomFlRTGCloud, AuditLoggerIntegrationTest, TransactionLogTest, PathHierarchyTokenizerFactoryTest, URLClassifyProcessorTest, TestReplicationHandlerBackup, MinimalSchemaTest, ZkCollectionPropsCachingTest, SolrRrdBackendFactoryTest, MoveReplicaHDFSFailoverTest, DirectSolrSpellCheckerTest, TestSimNodeLostTrigger, AnalysisErrorHandlingTest, DateFieldTest, AssignTest, BasicDistributedZk2Test, RollingRestartTest, TestSubQueryTransformerDistrib, ForceLeaderTest]
[junit4] Completed [393/910 (1!)] on J1 in 161.05s, 2 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 53859 lines...]
[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 474 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/474/
1 tests failed.
FAILED: org.apache.solr.cloud.DistribCursorPagingTest.test
Error Message:
Error from server at http://127.0.0.1:34839/collection1_shard1_replica_n1: Server error writing document id 1824 to the index
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34839/collection1_shard1_replica_n1: Server error writing document id 1824 to the index
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:681)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:266)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:248)
at org.apache.solr.client.solrj.impl.LBSolrClient.doRequest(LBSolrClient.java:370)
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:298)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.directUpdate(BaseCloudSolrClient.java:573)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.sendRequest(BaseCloudSolrClient.java:1060)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.requestWithRetryOnStaleState(BaseCloudSolrClient.java:918)
at org.apache.solr.client.solrj.impl.BaseCloudSolrClient.request(BaseCloudSolrClient.java:850)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:214)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:231)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.indexDoc(AbstractFullDistribZkTestBase.java:999)
at org.apache.solr.cloud.DistribCursorPagingTest.doRandomSortsOnLargeIndex(DistribCursorPagingTest.java:584)
at org.apache.solr.cloud.DistribCursorPagingTest.test(DistribCursorPagingTest.java:94)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 15040 lines...]
[junit4] Suite: org.apache.solr.cloud.DistribCursorPagingTest
[junit4] 2> 1979798 INFO (SUITE-DistribCursorPagingTest-seed#[A852740D0B0DF941]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/server/solr/configsets/_default/conf'
[junit4] 2> 1979799 INFO (SUITE-DistribCursorPagingTest-seed#[A852740D0B0DF941]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/data-dir-154-001
[junit4] 2> 1979799 WARN (SUITE-DistribCursorPagingTest-seed#[A852740D0B0DF941]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
[junit4] 2> 1979799 INFO (SUITE-DistribCursorPagingTest-seed#[A852740D0B0DF941]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1979800 INFO (SUITE-DistribCursorPagingTest-seed#[A852740D0B0DF941]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-9182 - causes OOM)
[junit4] 2> 1979800 INFO (SUITE-DistribCursorPagingTest-seed#[A852740D0B0DF941]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1979800 INFO (SUITE-DistribCursorPagingTest-seed#[A852740D0B0DF941]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1979804 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1979805 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
[junit4] 2> 1979805 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1979904 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer start zk server on port: 33105
[junit4] 2> 1979905 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:33105
[junit4] 2> 1979905 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:33105
[junit4] 2> 1979905 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 33105
[junit4] 2> 1979906 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1979909 INFO (zkConnectionManagerCallback-18568-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1979909 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1979912 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1979912 INFO (zkConnectionManagerCallback-18570-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1979912 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1979914 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-deeppaging.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1979915 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema-sorts.xml to /configs/conf1/schema.xml
[junit4] 2> 1979916 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1979917 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1979918 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1979919 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1979920 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1979922 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1979923 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1979924 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1979925 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1979927 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1980008 WARN (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 1980008 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 1980008 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 1980008 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1980010 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1980010 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1980010 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1980010 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2d63fa26{/,null,AVAILABLE}
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@8b7496a{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:37663}
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.s.Server Started @1980044ms
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/tempDir-001/control/data, replicaType=NRT, hostContext=/, hostPort=37663, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/control-001/cores}
[junit4] 2> 1980011 ERROR (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 1980011 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-15T03:43:49.398Z
[junit4] 2> 1980013 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1980014 INFO (zkConnectionManagerCallback-18572-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1980014 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1980117 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1980117 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/control-001/solr.xml
[junit4] 2> 1980120 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1980120 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1980122 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1980897 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1980898 WARN (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2058de0e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1980898 WARN (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2058de0e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1980900 WARN (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@6b6ed134[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1980900 WARN (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@6b6ed134[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1980901 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33105/solr
[junit4] 2> 1980902 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1980903 INFO (zkConnectionManagerCallback-18583-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1980903 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1981004 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1981005 INFO (zkConnectionManagerCallback-18585-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1981005 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1981105 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37663_
[junit4] 2> 1981105 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.c.Overseer Overseer (id=73932855719624708-127.0.0.1:37663_-n_0000000000) starting
[junit4] 2> 1981111 INFO (OverseerStateUpdate-73932855719624708-127.0.0.1:37663_-n_0000000000) [n:127.0.0.1:37663_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:37663_
[junit4] 2> 1981112 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37663_
[junit4] 2> 1981114 INFO (zkCallback-18584-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1981118 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 1981118 WARN (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 1981139 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1981158 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1981165 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1981165 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1981166 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [n:127.0.0.1:37663_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/control-001/cores
[junit4] 2> 1981177 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1981178 INFO (zkConnectionManagerCallback-18602-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1981178 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1981179 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1981180 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33105/solr ready
[junit4] 2> 1981184 INFO (qtp1169799349-37122) [n:127.0.0.1:37663_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37663_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1981188 INFO (OverseerThreadFactory-18592-thread-1-processing-n:127.0.0.1:37663_) [n:127.0.0.1:37663_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1981296 INFO (qtp1169799349-37126) [n:127.0.0.1:37663_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1981296 INFO (qtp1169799349-37126) [n:127.0.0.1:37663_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 1981302 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1982314 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1982326 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=sortable
[junit4] 2> 1982411 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema sortable/1.6 with uniqueid field id
[junit4] 2> 1982411 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from configset conf1, trusted=true
[junit4] 2> 1982411 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1982411 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1982414 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1982456 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1982456 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1982457 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1982457 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1982458 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@6a9b4db1
[junit4] 2> 1982460 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1982461 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1982461 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.h.ReplicationHandler Commits will be reserved for 10000 ms
[junit4] 2> 1982461 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1669534981132648448
[junit4] 2> 1982464 INFO (searcherExecutor-18604-thread-1-processing-n:127.0.0.1:37663_ x:control_collection_shard1_replica_n1 c:control_collection s:shard1) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher autowarm time: 0 ms
[junit4] 2> 1982473 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkShardTerms Successful update of terms at /collections/control_collection/terms/shard1 to Terms{values={core_node2=0}, version=0}
[junit4] 2> 1982473 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase make sure parent is created /collections/control_collection/leaders/shard1
[junit4] 2> 1982491 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1982491 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1982491 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:37663/control_collection_shard1_replica_n1/
[junit4] 2> 1982493 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1982493 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SyncStrategy http://127.0.0.1:37663/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1982494 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/control_collection/leaders/shard1/leader after winning as /collections/control_collection/leader_elect/shard1/election/73932855719624708-core_node2-n_0000000000
[junit4] 2> 1982497 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:37663/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1982600 INFO (zkCallback-18584-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982600 INFO (zkCallback-18584-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982601 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1982603 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1300
[junit4] 2> 1982605 INFO (qtp1169799349-37122) [n:127.0.0.1:37663_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 1982703 INFO (zkCallback-18584-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982703 INFO (zkCallback-18584-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982703 INFO (zkCallback-18584-thread-3) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1982703 INFO (qtp1169799349-37122) [n:127.0.0.1:37663_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37663_&wt=javabin&version=2} status=0 QTime=1518
[junit4] 2> 1982703 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Waiting to see 1 active replicas in collection: control_collection
[junit4] 2> 1982811 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1982812 INFO (zkConnectionManagerCallback-18613-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1982812 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1982814 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1982815 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33105/solr ready
[junit4] 2> 1982815 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1982816 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1982820 INFO (OverseerThreadFactory-18592-thread-2-processing-n:127.0.0.1:37663_) [n:127.0.0.1:37663_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1982821 INFO (OverseerCollectionConfigSetProcessor-73932855719624708-127.0.0.1:37663_-n_0000000000) [n:127.0.0.1:37663_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1983024 WARN (OverseerThreadFactory-18592-thread-2-processing-n:127.0.0.1:37663_) [n:127.0.0.1:37663_ ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1983026 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 45 seconds. Check all shard replicas
[junit4] 2> 1983027 INFO (qtp1169799349-37125) [n:127.0.0.1:37663_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=210
[junit4] 2> 1983028 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected: 2
[junit4] 2> 1983028 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
[junit4] 2> 1983028 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected: 2
[junit4] 2> 1983028 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
[junit4] 2> 1983028 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrCloudTestCase active slice count: 2 expected: 2
[junit4] 2> 1983028 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.SolrCloudTestCase active replica count: 0 expected replica count: 0
[junit4] 2> 1983028 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating jetty instances pullReplicaCount=0 numOtherReplicas=4
[junit4] 2> 1983109 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-1-001 of type NRT for shard2
[junit4] 2> 1983109 WARN (closeThreadPool-18614-thread-1) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 1983110 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 1983110 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 1983110 INFO (closeThreadPool-18614-thread-1) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1983113 INFO (closeThreadPool-18614-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1983113 INFO (closeThreadPool-18614-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1983113 INFO (closeThreadPool-18614-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1983116 INFO (closeThreadPool-18614-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@14c3e209{/,null,AVAILABLE}
[junit4] 2> 1983116 INFO (closeThreadPool-18614-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@518ba255{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:37295}
[junit4] 2> 1983116 INFO (closeThreadPool-18614-thread-1) [ ] o.e.j.s.Server Started @1983149ms
[junit4] 2> 1983116 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, hostPort=37295, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-1-001/cores}
[junit4] 2> 1983116 ERROR (closeThreadPool-18614-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1983117 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1983117 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 1983117 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1983117 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 1983117 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-15T03:43:52.504Z
[junit4] 2> 1983119 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1983121 INFO (zkConnectionManagerCallback-18616-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983121 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1983191 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-2-001 of type NRT for shard1
[junit4] 2> 1983191 WARN (closeThreadPool-18614-thread-2) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 1983191 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 1983191 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 1983191 INFO (closeThreadPool-18614-thread-2) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1983208 INFO (closeThreadPool-18614-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1983208 INFO (closeThreadPool-18614-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1983208 INFO (closeThreadPool-18614-thread-2) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1983212 INFO (closeThreadPool-18614-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@78a05f5a{/,null,AVAILABLE}
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@595f0abd{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:33791}
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.e.j.s.Server Started @1983246ms
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=33791, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-2-001/cores}
[junit4] 2> 1983213 ERROR (closeThreadPool-18614-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 1983213 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-15T03:43:52.600Z
[junit4] 2> 1983220 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1983222 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1983222 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-1-001/solr.xml
[junit4] 2> 1983224 INFO (zkConnectionManagerCallback-18618-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983224 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1983225 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1983225 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1983228 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1983274 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-3-001 of type NRT for shard2
[junit4] 2> 1983275 WARN (closeThreadPool-18614-thread-3) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 1983275 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 1983275 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 1983275 INFO (closeThreadPool-18614-thread-3) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1983278 INFO (closeThreadPool-18614-thread-3) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1983278 INFO (closeThreadPool-18614-thread-3) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1983278 INFO (closeThreadPool-18614-thread-3) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 1983278 INFO (closeThreadPool-18614-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@197dd0d4{/,null,AVAILABLE}
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@76914d2a{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:39995}
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.e.j.s.Server Started @1983311ms
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/tempDir-001/jetty3, solrconfig=solrconfig.xml, hostContext=/, hostPort=39995, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-3-001/cores}
[junit4] 2> 1983279 ERROR (closeThreadPool-18614-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 1983279 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-15T03:43:52.666Z
[junit4] 2> 1983280 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1983280 INFO (zkConnectionManagerCallback-18623-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983280 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1983327 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1983327 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-2-001/solr.xml
[junit4] 2> 1983330 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1983330 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1983332 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1983378 INFO (TEST-DistribCursorPagingTest.test-seed#[A852740D0B0DF941]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 4 in directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-4-001 of type NRT for shard1
[junit4] 2> 1983395 WARN (closeThreadPool-18614-thread-4) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 1983395 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 1983395 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 1983395 INFO (closeThreadPool-18614-thread-4) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 1983417 INFO (closeThreadPool-18614-thread-4) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1983417 INFO (closeThreadPool-18614-thread-4) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1983417 INFO (closeThreadPool-18614-thread-4) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 1983420 INFO (closeThreadPool-18614-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@48fdcdbe{/,null,AVAILABLE}
[junit4] 2> 1983420 INFO (closeThreadPool-18614-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@45d6a079{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:34839}
[junit4] 2> 1983420 INFO (closeThreadPool-18614-thread-4) [ ] o.e.j.s.Server Started @1983453ms
[junit4] 2> 1983420 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/tempDir-001/jetty4, solrconfig=solrconfig.xml, hostContext=/, hostPort=34839, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-4-001/cores}
[junit4] 2> 1983421 ERROR (closeThreadPool-18614-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1983421 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 1983421 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 1983421 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1983421 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 1983421 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-15T03:43:52.808Z
[junit4] 2> 1983427 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1983427 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-3-001/solr.xml
[junit4] 2> 1983430 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1983430 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1983432 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1983448 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1983473 INFO (zkConnectionManagerCallback-18628-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983473 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1983575 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1983575 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-4-001/solr.xml
[junit4] 2> 1983578 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1983578 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1983579 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1983921 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1983932 WARN (closeThreadPool-18614-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@1b413f8a[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983932 WARN (closeThreadPool-18614-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@1b413f8a[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983968 WARN (closeThreadPool-18614-thread-2) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2d81bc45[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983968 WARN (closeThreadPool-18614-thread-2) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2d81bc45[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1983969 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33105/solr
[junit4] 2> 1983976 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1983976 INFO (zkConnectionManagerCallback-18642-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1983976 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984061 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1984062 WARN (closeThreadPool-18614-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@12f1b179[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984062 WARN (closeThreadPool-18614-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@12f1b179[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984065 WARN (closeThreadPool-18614-thread-1) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@224389cd[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984065 WARN (closeThreadPool-18614-thread-1) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@224389cd[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984066 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33105/solr
[junit4] 2> 1984068 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1984068 INFO (zkConnectionManagerCallback-18650-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1984068 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984088 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1984088 INFO (zkConnectionManagerCallback-18652-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1984088 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984096 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1984113 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.ZkController Publish node=127.0.0.1:33791_ as DOWN
[junit4] 2> 1984115 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1984115 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33791_
[junit4] 2> 1984117 INFO (zkCallback-18612-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1984117 INFO (zkCallback-18584-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1984121 INFO (zkCallback-18651-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1984122 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 1984123 WARN (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 1984144 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1984175 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984188 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1984191 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984192 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984193 INFO (closeThreadPool-18614-thread-2) [n:127.0.0.1:33791_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-2-001/cores
[junit4] 2> 1984196 INFO (zkConnectionManagerCallback-18661-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1984196 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984205 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1984220 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.ZkController Publish node=127.0.0.1:37295_ as DOWN
[junit4] 2> 1984221 INFO (closeThreadPool-18614-thread-2) [ ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:33791_
[junit4] 2> 1984222 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1984222 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37295_
[junit4] 2> 1984223 INFO (zkCallback-18584-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1984223 INFO (zkCallback-18612-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1984224 INFO (zkCallback-18651-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1984227 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 1984227 WARN (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 1984235 INFO (zkCallback-18660-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1984254 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1984280 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984290 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984290 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984305 INFO (closeThreadPool-18614-thread-1) [n:127.0.0.1:37295_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-1-001/cores
[junit4] 2> 1984342 INFO (closeThreadPool-18614-thread-1) [ ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:37295_
[junit4] 2> 1984371 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1984371 WARN (closeThreadPool-18614-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@7419da62[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984371 WARN (closeThreadPool-18614-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@7419da62[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984373 WARN (closeThreadPool-18614-thread-3) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@3ab6c11e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984373 WARN (closeThreadPool-18614-thread-3) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@3ab6c11e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984374 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33105/solr
[junit4] 2> 1984375 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1984376 INFO (zkConnectionManagerCallback-18678-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1984376 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984480 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1984483 INFO (zkConnectionManagerCallback-18680-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1984483 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984494 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1984505 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.ZkController Publish node=127.0.0.1:39995_ as DOWN
[junit4] 2> 1984507 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1984507 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39995_
[junit4] 2> 1984508 INFO (zkCallback-18612-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1984508 INFO (zkCallback-18660-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1984508 INFO (zkCallback-18651-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1984510 INFO (zkCallback-18584-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1984513 INFO (zkCallback-18679-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1984520 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 1984520 WARN (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 1984534 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1984553 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984563 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984563 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984564 INFO (closeThreadPool-18614-thread-3) [n:127.0.0.1:39995_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-3-001/cores
[junit4] 2> 1984587 INFO (closeThreadPool-18614-thread-3) [ ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:39995_
[junit4] 2> 1984779 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 1984781 WARN (closeThreadPool-18614-thread-4) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@2152ad9a[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984781 WARN (closeThreadPool-18614-thread-4) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@2152ad9a[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984785 WARN (closeThreadPool-18614-thread-4) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@6edda579[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984785 WARN (closeThreadPool-18614-thread-4) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@6edda579[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 1984786 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33105/solr
[junit4] 2> 1984787 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1984787 INFO (zkConnectionManagerCallback-18696-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1984788 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984822 INFO (OverseerCollectionConfigSetProcessor-73932855719624708-127.0.0.1:37663_-n_0000000000) [n:127.0.0.1:37663_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1984890 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1984894 INFO (zkConnectionManagerCallback-18698-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1984894 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1984900 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 1984903 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.ZkController Publish node=127.0.0.1:34839_ as DOWN
[junit4] 2> 1984903 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1984903 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34839_
[junit4] 2> 1984904 INFO (zkCallback-18660-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1984904 INFO (zkCallback-18612-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1984904 INFO (zkCallback-18651-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1984905 INFO (zkCallback-18584-thread-4) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1984905 INFO (zkCallback-18679-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1984905 INFO (zkCallback-18697-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (5)
[junit4] 2> 1984907 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 1984908 WARN (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 1984924 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 1984952 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984962 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984962 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1984963 INFO (closeThreadPool-18614-thread-4) [n:127.0.0.1:34839_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-4-001/cores
[junit4] 2> 1984985 INFO (closeThreadPool-18614-thread-4) [ ] o.a.s.c.AbstractFullDistribZkTestBase waitForLiveNode: 127.0.0.1:34839_
[junit4] 2> 1984992 INFO (qtp1566610889-37194) [n:127.0.0.1:33791_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:39995_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1984992 INFO (qtp1566610889-37195) [n:127.0.0.1:33791_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:33791_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1984992 INFO (qtp1566610889-37196) [n:127.0.0.1:33791_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:34839_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1984992 INFO (qtp1566610889-37197) [n:127.0.0.1:33791_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:37295_&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1984998 INFO (qtp1566610889-37193) [n:127.0.0.1:33791_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 1984999 INFO (qtp1105804444-37220) [n:127.0.0.1:34839_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 1985000 INFO (qtp1437871163-37181) [n:127.0.0.1:37295_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 1985001 INFO (qtp1169799349-37122) [n:127.0.0.1:37663_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.control_collection.shard1.replica_n1:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 1985002 INFO (qtp1169799349-37124) [n:127.0.0.1:37663_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 1985003 INFO (qtp1924056999-37207) [n:127.0.0.1:39995_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 1985004 INFO (OverseerThreadFactory-18592-thread-4-processing-n:127.0.0.1:37663_) [n:127.0.0.1:37663_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:34839_ for creating new replica of shard shard1 for collection collection1
[junit4] 2> 1985004 INFO (OverseerThreadFactory-18592-thread-3-processing-n:127.0.0.1:37663_) [n:127.0.0.1:37663_ c:collection1 s:shard2 ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:39995_ for creating new replica of shard shard2 for collection collection1
[junit4] 2> 1985005 INFO (OverseerThreadFactory-18592-thread-4-processing-n:127.0.0.1:37663_) [n:127.0.0.1:37663_ c:collection1 s:shard1 ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 1985005 INFO (OverseerThreadFactory-18592-thread-3-processing-n:127.0.0.1:37663_) [n:127.0.0.1:37663_ c:collection1 s:shard2 ] o.a.s.c.a.c.AddReplicaCmd Returning CreateReplica command.
[junit4] 2> 1985008 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ x:collection1_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n1&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1985008 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ x:collection1_shard2_replica_n2 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n2&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1986019 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ c:collection1 s:shard1 x:collection1_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1986020 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 1986031 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ c:collection1 s:shard1 x:collection1_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=sortable
[junit4] 2> 1986036 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.s.IndexSchema Schema name=sortable
[junit4] 2> 1986146 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.s.IndexSchema Loaded schema sortable/1.6 with uniqueid field id
[junit4] 2> 1986146 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n2' using configuration from configset conf1, trusted=true
[junit4] 2> 1986146 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n2' (registry 'solr.core.collection1.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1986147 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.c.SolrCore [[collection1_shard2_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-3-001/cores/collection1_shard2_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-3-001/cores/collection1_shard2_replica_n2/data/]
[junit4] 2> 1986149 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1986149 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ c:collection1 s:shard1 x:collection1_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema sortable/1.6 with uniqueid field id
[junit4] 2> 1986149 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ c:collection1 s:shard1 x:collection1_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n1' using configuration from configset conf1, trusted=true
[junit4] 2> 1986150 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ c:collection1 s:shard1 x:collection1_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n1' (registry 'solr.core.collection1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11fffabb
[junit4] 2> 1986150 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ c:collection1 s:shard1 x:collection1_shard1_replica_n1 ] o.a.s.c.SolrCore [[collection1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-4-001/cores/collection1_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001/shard-4-001/cores/collection1_shard1_replica_n1/data/]
[junit4] 2> 1986152 INFO (qtp1105804444-37223) [n:127.0.0.1:34839_ c:collection1 s:shard1 x:collection1_shard1_replica_n1 ] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=48, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 1986195 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1986195 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1986196 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1986196 INFO (qtp1924056999-37209) [n:127.0.0.1:39995_ c:collection1 s:shard2 x:collection1_shard2_replica_n2 ] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1986197 INFO (qtp1
[...truncated too long message...]
apache.lucene.codecs.StoredFieldsWriter.merge(StoredFieldsWriter.java:134)
[junit4] > at org.apache.lucene.index.SegmentMerger.mergeFields(SegmentMerger.java:228)
[junit4] > at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:105)
[junit4] > at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4458)
[junit4] > at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:4052)
[junit4] > at org.apache.solr.update.SolrIndexWriter.merge(SolrIndexWriter.java:201)
[junit4] > at org.apache.lucene.index.IndexWriter$IndexWriterMergeSource.merge(IndexWriter.java:5612)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler.doMerge(ConcurrentMergeScheduler.java:624)
[junit4] > at org.apache.lucene.index.ConcurrentMergeScheduler$MergeThread.run(ConcurrentMergeScheduler.java:682)
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.DistribCursorPagingTest_A852740D0B0DF941-001
[junit4] 2> Jun 15, 2020 3:44:41 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene86): {bool=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), collation_en_primary_first=PostingsFormat(name=LuceneVarGapFixedInterval), currency_first____amount_raw=PostingsFormat(name=LuceneVarGapDocFreqInterval), str_last=PostingsFormat(name=LuceneFixedGap), currency_first____currency=PostingsFormat(name=LuceneVarGapDocFreqInterval), uuid=PostingsFormat(name=LuceneVarGapFixedInterval), enum_first=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), str_first=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), icu_collation_en_primary_first=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneVarGapFixedInterval), currency____amount_raw=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), collation_en_primary_last=PostingsFormat(name=LuceneVarGapDocFreqInterval), currency_last____currency=PostingsFormat(name=LuceneVarGapFixedInterval), uuid_first=PostingsFormat(name=LuceneFixedGap), bool_first=PostingsFormat(name=LuceneVarGapDocFreqInterval), _root_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), bool_last=PostingsFormat(name=LuceneVarGapFixedInterval), uuid_last=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), enum=PostingsFormat(name=LuceneVarGapDocFreqInterval), str=PostingsFormat(name=LuceneVarGapDocFreqInterval), enum_dv_last=PostingsFormat(name=LuceneVarGapFixedInterval), enum_last=PostingsFormat(name=LuceneFixedGap), icu_collation_en_primary_last=PostingsFormat(name=LuceneVarGapDocFreqInterval), currency____currency=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene84)), currency_last____amount_raw=PostingsFormat(name=LuceneVarGapDocFreqInterval), enum_dv_first=PostingsFormat(name=LuceneVarGapDocFreqInterval), enum_dv=PostingsFormat(name=LuceneFixedGap)}, docValues:{date=DocValuesFormat(name=Lucene80), collation_en_primary_dv_first=DocValuesFormat(name=Lucene80), collation_en_primary_dv=DocValuesFormat(name=Asserting), bin=DocValuesFormat(name=Direct), date_dv_last=DocValuesFormat(name=Lucene80), float=DocValuesFormat(name=Lucene80), bin_last=DocValuesFormat(name=Lucene80), int_dv=DocValuesFormat(name=Lucene80), str_dv_first=DocValuesFormat(name=Asserting), icu_collation_en_primary_dv=DocValuesFormat(name=Asserting), float_dv=DocValuesFormat(name=Asserting), long_dv_last=DocValuesFormat(name=Lucene80), bin_first=DocValuesFormat(name=Lucene80), collation_en_primary_dv_last=DocValuesFormat(name=Lucene80), date_last=DocValuesFormat(name=Asserting), long_dv_first=DocValuesFormat(name=Lucene80), double=DocValuesFormat(name=Direct), float_dv_last=DocValuesFormat(name=Lucene80), bin_dv_last=DocValuesFormat(name=Asserting), uuid_dv_first=DocValuesFormat(name=Asserting), double_dv=DocValuesFormat(name=Lucene80), enum_dv_last=DocValuesFormat(name=Direct), str_dv=DocValuesFormat(name=Lucene80), date_first=DocValuesFormat(name=Direct), bin_dv_first=DocValuesFormat(name=Asserting), enum_dv=DocValuesFormat(name=Lucene80), uuid_dv=DocValuesFormat(name=Lucene80), icu_collation_en_primary_dv_first=DocValuesFormat(name=Lucene80), date_dv=DocValuesFormat(name=Direct), float_dv_first=DocValuesFormat(name=Lucene80), double_dv_last=DocValuesFormat(name=Direct), uuid_dv_last=DocValuesFormat(name=Asserting), date_dv_first=DocValuesFormat(name=Lucene80), double_last=DocValuesFormat(name=Lucene80), int_last=DocValuesFormat(name=Lucene80), long=DocValuesFormat(name=Lucene80), str_dv_last=DocValuesFormat(name=Direct), double_first=DocValuesFormat(name=Lucene80), icu_collation_en_primary_dv_last=DocValuesFormat(name=Lucene80), float_first=DocValuesFormat(name=Direct), long_first=DocValuesFormat(name=Direct), int_dv_first=DocValuesFormat(name=Direct), long_last=DocValuesFormat(name=Asserting), double_dv_first=DocValuesFormat(name=Direct), int=DocValuesFormat(name=Direct), int_first=DocValuesFormat(name=Lucene80), int_dv_last=DocValuesFormat(name=Direct), _version_=DocValuesFormat(name=Lucene80), enum_dv_first=DocValuesFormat(name=Asserting), float_last=DocValuesFormat(name=Direct), long_dv=DocValuesFormat(name=Asserting), bin_dv=DocValuesFormat(name=Lucene80)}, maxPointsInLeafNode=1801, maxMBSortInHeap=5.689669565461712, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@1c4022d9), locale=ko, timezone=Atlantic/Madeira
[junit4] 2> NOTE: Linux 4.15.0-54-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=4,threads=1,free=163048360,total=519569408
[junit4] 2> NOTE: All tests run in this JVM: [TestFastOutputStream, TestGraphTermsQParserPlugin, TestConfig, TestHighFrequencyDictionaryFactory, ProtectedTermFilterFactoryTest, TestSurroundQueryParser, HttpPartitionTest, TestLuceneMatchVersion, TestUnifiedSolrHighlighterWithoutStoredId, TestSmileRequest, TestTolerantSearch, TestNestedUpdateProcessor, AnalyticsMergeStrategyTest, SliceStateTest, RAMDirectoryFactoryTest, ConcurrentDeleteAndCreateCollectionTest, TestOverriddenPrefixQueryForCustomFieldType, TestAddFieldRealTimeGet, SynonymTokenizerTest, TriggerIntegrationTest, CdcrBidirectionalTest, TestFileDictionaryLookup, UUIDUpdateProcessorFallbackTest, SystemInfoHandlerTest, SuggestComponentTest, TestCloudManagedSchema, TestExactStatsCache, TestCollationFieldDocValues, TestHdfsUpdateLog, TestDocumentBuilder, ZookeeperStatusHandlerTest, AutoAddReplicasIntegrationTest, TestChildDocTransformer, SolrRequestParserTest, SOLR749Test, TestStandardQParsers, TestClassicSimilarityFactory, TestLegacyFieldReuse, RankQueryTest, TestDelegationWithHadoopAuth, DeleteLastCustomShardedReplicaTest, TestBinaryResponseWriter, TermsComponentTest, SpellCheckCollatorWithCollapseTest, PolyFieldTest, MetricsConfigTest, StressHdfsTest, TestDistributedGrouping, TestCollectionAPI, BlockJoinFacetSimpleTest, DistributedDebugComponentTest, TestDefaultStatsCache, TestSolrCachePerf, LegacyCloudClusterPropTest, TestLocalFSCloudBackupRestore, TestConfigsApi, HdfsLockFactoryTest, BlockDirectoryTest, TestExternalRoleRuleBasedAuthorizationPlugin, TestSSLRandomization, MultiThreadedOCPTest, SolrIndexSearcherTest, TestSimExecutePlanAction, TestCloudPivotFacet, SecurityConfHandlerTest, TestConfigSetsAPI, DistributedFacetPivotSmallTest, TestRTGBase, TestStressInPlaceUpdates, TestBulkSchemaAPI, PKIAuthenticationIntegrationTest, ReplicationFactorTest, TestMissingGroups, SchemaVersionSpecificBehaviorTest, TestZkChroot, NodeLostTriggerTest, PeerSyncWithBufferUpdatesTest, AtomicUpdatesTest, TestFieldTypeCollectionResource, FacetPivotSmallTest, TestXIncludeConfig, SortByFunctionTest, SpellCheckComponentTest, TestExtendedDismaxParser, HdfsRestartWhileUpdatingTest, TestIBSimilarityFactory, NodeAddedTriggerIntegrationTest, DistributedVersionInfoTest, OverseerTest, DynamicMapsTest, TestRTimerTree, SubstringBytesRefFilterTest, TestSolrJ, TestDynamicFieldResource, HdfsDirectoryFactoryTest, TestPackages, PeerSyncWithLeaderTest, AsyncCallRequestStatusResponseTest, TestUtilizeNode, XsltUpdateRequestHandlerTest, CoreAdminCreateDiscoverTest, OrderedExecutorTest, TestFilteredDocIdSet, TestRawTransformer, TestMultiValuedNumericRangeQuery, TestComplexPhraseLeadingWildcard, ShardRoutingCustomTest, ScheduledMaintenanceTriggerTest, RuleEngineTest, TestTlogReplica, DOMUtilTest, HighlighterConfigTest, CreateCollectionCleanupTest, TestHdfsBackupRestoreCore, TestSolrConfigHandlerConcurrent, ParsingFieldUpdateProcessorsTest, TestDirectoryFactory, SolrCloudReportersTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestInitQParser, SearchRateTriggerIntegrationTest, ConcurrentCreateRoutedAliasTest, AlternateDirectoryTest, TestNestedDocsSort, DeleteShardTest, ForceLeaderWithTlogReplicasTest, TestSimUtils, TestFieldCacheSortRandom, ZkShardTermsTest, TestManagedSynonymFilterFactory, DefaultValueUpdateProcessorTest, TestReRankQParserPlugin, DistributedSpellCheckComponentTest, CdcrUpdateLogTest, SolrCmdDistributorTest, TestRandomRequestDistribution, DistributedFacetSimpleRefinementLongTailTest, ByteBuffersDirectoryFactoryTest, SolrMetricReporterTest, TestCloudDeleteByQuery, TestPseudoReturnFields, TestSchemalessBufferedUpdates, TestNumericTokenStream, CoreSorterTest, CustomHighlightComponentTest, TestSimpleTrackingShardHandler, TestCharFilters, TestWordDelimiterFilterFactory, TokenizerChainTest, TestLuceneIndexBackCompat, AssignBackwardCompatibilityTest, BasicDistributedZkTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionPropsTest, CreateRoutedAliasTest, DistribCursorPagingTest]
[junit4] Completed [487/910 (1!)] on J2 in 52.79s, 1 test, 1 error <<< FAILURES!
[...truncated 53500 lines...]
[JENKINS] Lucene-Solr-NightlyTests-8.x - Build # 473 - Still
Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-8.x/473/
2 tests failed.
FAILED: org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test
Error Message:
.response.numFound:36!=67
Stack Trace:
junit.framework.AssertionFailedError: .response.numFound:36!=67
at __randomizedtesting.SeedInfo.seed([C744DD558487F2BA:4F10E28F2A7B9F42]:0)
at junit.framework.Assert.fail(Assert.java:57)
at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:1006)
at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:1033)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:679)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:655)
at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:634)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkQueries(AbstractFullDistribZkTestBase.java:1097)
at org.apache.solr.cloud.BasicDistributedZk2Test.test(BasicDistributedZk2Test.java:96)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:1090)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:1061)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
Error Message:
ObjectTracker found 4 object(s) that were not released!!! [NRTCachingDirectory, InternalHttpClient, SolrCore, NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:802) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:682) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:334) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1077) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:542) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:980) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) expected null, but was:<ObjectTracker found 4 object(s) that were not released!!! [NRTCachingDirectory, InternalHttpClient, SolrCore, NRTCachingDirectory] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98) at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:802) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339) at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348) at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230) at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272) at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420) at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252) at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:682) at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:334) at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317) at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1077) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354) at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:542) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:980) at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906) at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322) at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231) at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95) at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367) at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397) at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) at org.eclipse.jetty.server.Server.handle(Server.java:500) at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383) at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375) at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806) at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938) at java.lang.Thread.run(Thread.java:748) >
Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [NRTCachingDirectory, InternalHttpClient, SolrCore, NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:802)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:682)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:334)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1077)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:542)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:980)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
expected null, but was:<ObjectTracker found 4 object(s) that were not released!!! [NRTCachingDirectory, InternalHttpClient, SolrCore, NRTCachingDirectory]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:98)
at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:802)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:988)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.http.impl.client.InternalHttpClient
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:339)
at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:348)
at org.apache.solr.handler.IndexFetcher.createHttpClient(IndexFetcher.java:230)
at org.apache.solr.handler.IndexFetcher.<init>(IndexFetcher.java:272)
at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:420)
at org.apache.solr.cloud.RecoveryStrategy.replicate(RecoveryStrategy.java:252)
at org.apache.solr.cloud.RecoveryStrategy.doSyncOrReplicateRecovery(RecoveryStrategy.java:682)
at org.apache.solr.cloud.RecoveryStrategy.doRecovery(RecoveryStrategy.java:334)
at org.apache.solr.cloud.RecoveryStrategy.run(RecoveryStrategy.java:317)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:210)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.core.SolrCore
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1077)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.lucene.store.NRTCachingDirectory
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:354)
at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:542)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:980)
at org.apache.solr.core.SolrCore.<init>(SolrCore.java:906)
at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1322)
at org.apache.solr.core.CoreContainer.create(CoreContainer.java:1231)
at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:95)
at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:367)
at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:397)
at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:181)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:214)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:845)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:811)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:563)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:415)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:345)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:166)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1604)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:545)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1610)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:233)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1300)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:188)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:485)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1580)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:186)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1215)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.rewrite.handler.RewriteHandler.handle(RewriteHandler.java:322)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:717)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127)
at org.eclipse.jetty.server.Server.handle(Server.java:500)
at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:383)
at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:547)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:375)
at org.eclipse.jetty.server.HttpChannel.run(HttpChannel.java:335)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:806)
at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:938)
at java.lang.Thread.run(Thread.java:748)
>
at __randomizedtesting.SeedInfo.seed([C744DD558487F2BA]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.failNotNull(Assert.java:755)
at org.junit.Assert.assertNull(Assert.java:737)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:334)
at sun.reflect.GeneratedMethodAccessor83.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:901)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13874 lines...]
[junit4] Suite: org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest
[junit4] 2> 709897 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.s.SolrTestCase Setting 'solr.default.confdir' system property to test-framework derived value of '/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/server/solr/configsets/_default/conf'
[junit4] 2> 709898 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 709898 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.s.SolrTestCaseJ4 Created dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_C744DD558487F2BA-001/data-dir-59-001
[junit4] 2> 709898 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 709898 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 709900 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 709900 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 1> Formatting using clusterid: testClusterID
[junit4] 2> 709942 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 709944 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 709945 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 709945 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 709945 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 709945 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2eca8d68{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 710072 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@1d8a3f42{hdfs,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/jetty-lucene2-us-west_apache_org-42343-hadoop-hdfs-3_2_0-tests_jar-_-any-4382989973434374126.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
[junit4] 2> 710072 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5299d030{HTTP/1.1, (http/1.1)}{lucene2-us-west.apache.org:42343}
[junit4] 2> 710072 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.Server Started @710108ms
[junit4] 2> 710126 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 710128 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 710128 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 710128 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 710128 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 710129 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61e95412{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 710252 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@1b65328{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-42960-hadoop-hdfs-3_2_0-tests_jar-_-any-3079539195655486448.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
[junit4] 2> 710252 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3cd10789{HTTP/1.1, (http/1.1)}{localhost:42960}
[junit4] 2> 710252 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.Server Started @710288ms
[junit4] 2> 710297 WARN (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
[junit4] 2> 710298 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 710300 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 710300 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 710300 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 710300 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@74e665d3{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,AVAILABLE}
[junit4] 2> 710363 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe8ff4fa1a49ee6e1: Processing first storage report for DS-a15c0327-b9d3-4e8c-8afe-16d561131193 from datanode 20db34cf-4024-49d3-8195-c1d9744b3cd9
[junit4] 2> 710363 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe8ff4fa1a49ee6e1: from storage DS-a15c0327-b9d3-4e8c-8afe-16d561131193 node DatanodeRegistration(127.0.0.1:36368, datanodeUuid=20db34cf-4024-49d3-8195-c1d9744b3cd9, infoPort=36024, infoSecurePort=0, ipcPort=37171, storageInfo=lv=-57;cid=testClusterID;nsid=1376595093;c=1592101514054), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 710363 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe8ff4fa1a49ee6e1: Processing first storage report for DS-1ff82f81-957d-4de6-998b-6f6ff858794c from datanode 20db34cf-4024-49d3-8195-c1d9744b3cd9
[junit4] 2> 710363 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xe8ff4fa1a49ee6e1: from storage DS-1ff82f81-957d-4de6-998b-6f6ff858794c node DatanodeRegistration(127.0.0.1:36368, datanodeUuid=20db34cf-4024-49d3-8195-c1d9744b3cd9, infoPort=36024, infoSecurePort=0, ipcPort=37171, storageInfo=lv=-57;cid=testClusterID;nsid=1376595093;c=1592101514054), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 710432 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Started o.e.j.w.WebAppContext@36c54a16{datanode,/,file:///home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/jetty-localhost-38522-hadoop-hdfs-3_2_0-tests_jar-_-any-3172059123467877898.dir/webapp/,AVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
[junit4] 2> 710432 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6458798a{HTTP/1.1, (http/1.1)}{localhost:38522}
[junit4] 2> 710433 INFO (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.Server Started @710468ms
[junit4] 2> 710511 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xb7c444b0353d59c2: Processing first storage report for DS-7e4a69bc-f24f-406d-9606-6deca3ed64b0 from datanode 7167af56-5df1-468b-9bfd-d6e527d8b0a6
[junit4] 2> 710511 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xb7c444b0353d59c2: from storage DS-7e4a69bc-f24f-406d-9606-6deca3ed64b0 node DatanodeRegistration(127.0.0.1:44585, datanodeUuid=7167af56-5df1-468b-9bfd-d6e527d8b0a6, infoPort=46726, infoSecurePort=0, ipcPort=43538, storageInfo=lv=-57;cid=testClusterID;nsid=1376595093;c=1592101514054), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 710511 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xb7c444b0353d59c2: Processing first storage report for DS-8f314b11-5998-46f2-a6f2-b39561596556 from datanode 7167af56-5df1-468b-9bfd-d6e527d8b0a6
[junit4] 2> 710511 INFO (Block report processor) [ ] BlockStateChange BLOCK* processReport 0xb7c444b0353d59c2: from storage DS-8f314b11-5998-46f2-a6f2-b39561596556 node DatanodeRegistration(127.0.0.1:44585, datanodeUuid=7167af56-5df1-468b-9bfd-d6e527d8b0a6, infoPort=46726, infoSecurePort=0, ipcPort=43538, storageInfo=lv=-57;cid=testClusterID;nsid=1376595093;c=1592101514054), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0
[junit4] 2> 710605 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 710605 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer client port: 0.0.0.0/0.0.0.0:0
[junit4] 2> 710605 INFO (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 710705 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer start zk server on port: 40158
[junit4] 2> 710705 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer waitForServerUp: 127.0.0.1:40158
[junit4] 2> 710705 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:40158
[junit4] 2> 710705 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 40158
[junit4] 2> 710707 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 710710 INFO (zkConnectionManagerCallback-5178-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 710710 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 710713 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 710714 INFO (zkConnectionManagerCallback-5180-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 710714 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 710716 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 710718 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 710720 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 710721 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 710722 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 710723 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 710725 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 710726 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 710727 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 710729 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 710730 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkTestServer put /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 710731 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 710845 WARN (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.h.g.GzipHandler minGzipSize of 0 is inefficient for short content, break even is size 23
[junit4] 2> 710845 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.s.e.JettySolrRunner Start Jetty (configured port=0, binding port=0)
[junit4] 2> 710845 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.s.e.JettySolrRunner Trying to start Jetty on port 0 try number 2 ...
[junit4] 2> 710845 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.Server jetty-9.4.27.v20200227; built: 2020-02-27T18:37:21.340Z; git: a304fd9f351f337e7c0e2a7c28878dd536149c6c; jvm 1.8.0_191-b12
[junit4] 2> 710846 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 710846 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 710846 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 710846 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@41d9bd2a{/,null,AVAILABLE}
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@236367c9{HTTP/1.1, (http/1.1, h2c)}{127.0.0.1:37387}
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.s.Server Started @710882ms
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://lucene2-us-west.apache.org:38670/hdfs__lucene2-us-west.apache.org_38670__home_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-8.x_checkout_solr_build_solr-core_test_J1_temp_solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_C744DD558487F2BA-001_tempDir-002_control_data, replicaType=NRT, hostContext=/, hostPort=37387, coreRootDirectory=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_C744DD558487F2BA-001/control-001/cores}
[junit4] 2> 710847 ERROR (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.6.0
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr
[junit4] 2> 710847 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2020-06-14T02:25:14.986Z
[junit4] 2> 710848 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 710850 INFO (zkConnectionManagerCallback-5182-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 710850 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 710951 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 710951 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_C744DD558487F2BA-001/control-001/solr.xml
[junit4] 2> 710955 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 710955 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 710956 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@11746b0b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 711504 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.h.c.HttpShardHandlerFactory Host whitelist initialized: WhitelistHostChecker [whitelistHosts=null, whitelistHostCheckingEnabled=false]
[junit4] 2> 711508 WARN (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5bda243e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 711508 WARN (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5bda243e[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 711511 WARN (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.u.s.S.config Trusting all certificates configured for Client@5ac40118[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 711511 WARN (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.e.j.u.s.S.config No Client EndPointIdentificationAlgorithm configured for Client@5ac40118[provider=null,keyStore=null,trustStore=null]
[junit4] 2> 711513 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40158/solr
[junit4] 2> 711513 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 711515 INFO (zkConnectionManagerCallback-5193-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 711515 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 711617 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 711619 INFO (zkConnectionManagerCallback-5195-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 711619 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 711704 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:37387_
[junit4] 2> 711705 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.c.Overseer Overseer (id=72967040408354820-127.0.0.1:37387_-n_0000000000) starting
[junit4] 2> 711710 INFO (OverseerStateUpdate-72967040408354820-127.0.0.1:37387_-n_0000000000) [n:127.0.0.1:37387_ ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:37387_
[junit4] 2> 711711 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37387_
[junit4] 2> 711713 INFO (zkCallback-5194-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 711715 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.p.PackageLoader /packages.json updated to version -1
[junit4] 2> 711715 WARN (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.c.CoreContainer Not all security plugins configured! authentication=disabled authorization=disabled. Solr is only as secure as you make it. Consider configuring authentication/authorization before exposing Solr to users internal or external. See https://s.apache.org/solrsecurity for more info
[junit4] 2> 711743 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 711773 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11746b0b
[junit4] 2> 711786 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11746b0b
[junit4] 2> 711786 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11746b0b
[junit4] 2> 711788 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [n:127.0.0.1:37387_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_C744DD558487F2BA-001/control-001/cores
[junit4] 2> 711801 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 711802 INFO (zkConnectionManagerCallback-5212-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 711802 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 711804 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 711805 INFO (TEST-HdfsChaosMonkeySafeLeaderTest.test-seed#[C744DD558487F2BA]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40158/solr ready
[junit4] 2> 711807 INFO (qtp578925224-7636) [n:127.0.0.1:37387_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:37387_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 711810 INFO (OverseerThreadFactory-5202-thread-1-processing-n:127.0.0.1:37387_) [n:127.0.0.1:37387_ ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 711918 INFO (qtp578925224-7635) [n:127.0.0.1:37387_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 711918 INFO (qtp578925224-7635) [n:127.0.0.1:37387_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CONTAINER.fs.totalSpace,CONTAINER.cores&wt=javabin&version=2&group=solr.node} status=0 QTime=0
[junit4] 2> 711922 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ x:control_collection_shard1_replica_n1 ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 712937 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.6.0
[junit4] 2> 712956 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Schema name=test
[junit4] 2> 713040 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 713062 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from configset conf1, trusted=true
[junit4] 2> 713062 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@11746b0b
[junit4] 2> 713072 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory solr.hdfs.home=hdfs://lucene2-us-west.apache.org:38670/solr_hdfs_home
[junit4] 2> 713073 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory Solr Kerberos Authentication disabled
[junit4] 2> 713073 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_C744DD558487F2BA-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[hdfs://lucene2-us-west.apache.org:38670/solr_hdfs_home/control_collection/core_node2/data/]
[junit4] 2> 713075 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory creating directory factory for path hdfs://lucene2-us-west.apache.org:38670/solr_hdfs_home/control_collection/core_node2/data/snapshot_metadata
[junit4] 2> 713091 INFO (qtp578925224-7637) [n:127.0.0.1:37387_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1 ] o.a.s.c.HdfsDirectoryFactory Number of slabs of block cache [1] with direct memory allocation set to [true]
[junit4] 2> 713091 INFO (qtp578925224-763
[...truncated too long message...]
(SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.a.h.h.s.d.DirectoryScanner DirectoryScanner: shutdown has been called
[junit4] 2> 6882910 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.w.WebAppContext@7add7e88{datanode,/,null,UNAVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/datanode}
[junit4] 2> 6882910 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@a585cb9{HTTP/1.1, (http/1.1)}{localhost:0}
[junit4] 2> 6882910 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 6882911 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2cb60c59{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
[junit4] 2> 6882915 WARN (BP-709204245-127.0.0.1-1592107449005 heartbeating to lucene2-us-west.apache.org/127.0.0.1:34714) [ ] o.a.h.h.s.d.IncrementalBlockReportManager IncrementalBlockReportManager interrupted
[junit4] 2> 6882916 WARN (BP-709204245-127.0.0.1-1592107449005 heartbeating to lucene2-us-west.apache.org/127.0.0.1:34714) [ ] o.a.h.h.s.d.DataNode Ending block pool service for: Block pool BP-709204245-127.0.0.1-1592107449005 (Datanode Uuid 21770ddd-ba11-4ba7-bfdb-0e5bc9ab2238) service to lucene2-us-west.apache.org/127.0.0.1:34714
[junit4] 2> 6882946 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.w.WebAppContext@34f48a7d{hdfs,/,null,UNAVAILABLE}{jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/hdfs}
[junit4] 2> 6882947 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@2b1e294b{HTTP/1.1, (http/1.1)}{lucene2-us-west.apache.org:0}
[junit4] 2> 6882947 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.session node0 Stopped scavenging
[junit4] 2> 6882947 INFO (SUITE-HdfsBasicDistributedZk2Test-seed#[C744DD558487F2BA]-worker) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3da9dcc7{static,/static,jar:file:/home/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-3.2.0-tests.jar!/webapps/static,UNAVAILABLE}
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-8.x/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsBasicDistributedZk2Test_C744DD558487F2BA-001
[junit4] 2> Jun 14, 2020 4:08:07 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 32 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene86): {rnd_b=PostingsFormat(name=LuceneVarGapFixedInterval), SubjectTerms_mfacet=PostingsFormat(name=LuceneFixedGap), oddField_s=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=MockRandom), _root_=PostingsFormat(name=LuceneFixedGap), foo_b=PostingsFormat(name=Asserting), id=PostingsFormat(name=LuceneVarGapFixedInterval), text=PostingsFormat(name=Asserting)}, docValues:{other_tl1=DocValuesFormat(name=Lucene80), range_facet_l_dv=DocValuesFormat(name=Lucene80), n_l1=DocValuesFormat(name=Asserting), intDefault=DocValuesFormat(name=Lucene80), n_dt1=DocValuesFormat(name=Lucene80), a_i1=DocValuesFormat(name=Asserting), n_td1=DocValuesFormat(name=Lucene80), n_d1=DocValuesFormat(name=Asserting), range_facet_l=DocValuesFormat(name=Asserting), n_f1=DocValuesFormat(name=Lucene80), n_ti1=DocValuesFormat(name=Lucene80), n_tl1=DocValuesFormat(name=Lucene80), _version_=DocValuesFormat(name=Lucene80), n_tf1=DocValuesFormat(name=Asserting), n_tdt1=DocValuesFormat(name=Lucene80), id_i1=DocValuesFormat(name=Lucene80), foo_d=DocValuesFormat(name=Lucene80), range_facet_i_dv=DocValuesFormat(name=Asserting), intDvoDefault=DocValuesFormat(name=Lucene80), foo_f=DocValuesFormat(name=Direct), timestamp=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=73, maxMBSortInHeap=5.7856090973808385, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@28b7a4f0), locale=ar-IQ, timezone=Etc/GMT+11
[junit4] 2> NOTE: Linux 4.4.0-170-generic amd64/Oracle Corporation 1.8.0_191 (64-bit)/cpus=8,threads=4,free=294391056,total=517472256
[junit4] 2> NOTE: All tests run in this JVM: [TestExceedMaxTermLength, TestRequestForwarding, TestDeprecatedFilters, ReindexCollectionTest, SoftAutoCommitTest, CdcrVersionReplicationTest, TestStressRecovery, TestCursorMarkWithoutUniqueKey, TestShortCircuitedRequests, SimpleFacetsTest, SolrIndexMetricsTest, TestSubQueryTransformerDistrib, AuditLoggerIntegrationTest, TestSolrTestCaseJ4, TestQueryUtils, TestLeaderElectionWithEmptyReplica, HdfsThreadLeakTest, TestLazyCores, CurrencyRangeFacetCloudTest, TestFieldCacheWithThreads, TransactionLogTest, TestRecovery, HttpPartitionOnCommitTest, SignificantTermsQParserPluginTest, SpatialFilterTest, MetricsHistoryWithAuthIntegrationTest, SplitByPrefixTest, VersionInfoTest, SyncSliceTest, BadCopyFieldTest, TestFuzzyAnalyzedSuggestions, InfoHandlerTest, TestBulkSchemaConcurrent, FieldAnalysisRequestHandlerTest, HdfsChaosMonkeySafeLeaderTest, DistributedQueueTest, OverseerTaskQueueTest, TestRandomFaceting, TestFoldingMultitermQuery, SaslZkACLProviderTest, TestDynamicFieldNamesIndexCorrectly, TestDynamicLoadingUrl, TemplateUpdateProcessorTest, TestLegacyField, TestManagedStopFilterFactory, TestSha256AuthenticationProvider, TriLevelCompositeIdRoutingTest, NumberUtilsTest, TestSort, XmlUpdateRequestHandlerTest, DistanceFunctionTest, StatsReloadRaceTest, TestExactSharedStatsCache, DistributedFacetExistsSmallTest, TestCollectionAPIs, ZkNodePropsTest, RegexBoostProcessorTest, UninvertDocValuesMergePolicyTest, TestStressCloudBlindAtomicUpdates, DebugComponentTest, NestedAtomicUpdateTest, TestSimpleQParserPlugin, HighlighterTest, TestRandomFlRTGCloud, SharedFSAutoReplicaFailoverTest, TestRestoreCore, DisMaxRequestHandlerTest, DistanceUnitsTest, MoreLikeThisComponentTest, BasicDistributedZk2Test, SchemaApiFailureTest, SuggestComponentContextFilterQueryTest, ChaosMonkeyNothingIsSafeTest, PeerSyncWithLeaderAndIndexFingerprintCachingTest, FastVectorHighlighterTest, TestDocTermOrdsUninvertLimit, CloudExitableDirectoryReaderTest, SuggesterWFSTTest, TestInfoStreamLogging, TestDFRSimilarityFactory, ZkCLITest, SuggesterFSTTest, TestEmbeddedSolrServerAdminHandler, TestJsonRangeFacets, BlockCacheTest, TestRawResponseWriter, SystemLogListenerTest, RestoreTriggerStateTest, TestReplicationHandlerBackup, TestAuthorizationFramework, TestCollapseQParserPlugin, SolrRrdBackendFactoryTest, TestBlobHandler, MetricUtilsTest, ShowFileRequestHandlerTest, DirectSolrSpellCheckerTest, CollectionsAPISolrJTest, ResponseBuilderTest, TestTrieFacet, TestFieldCacheSort, SpatialRPTFieldTypeTest, ZkDynamicConfigTest, TestSolrQueryResponse, TestComponentsName, TestSystemCollAutoCreate, HdfsRecoveryZkTest, CloudReplicaSourceTest, NodeMutatorTest, TestConfigSetImmutable, BlockJoinFacetRandomTest, BaseCdcrDistributedZkTest, TestJavabinTupleStreamParser, TestDynamicFieldCollectionResource, TestIndexingPerformance, DeleteInactiveReplicaTest, BigEndianAscendingWordDeserializerTest, TestNumericRangeQuery64, AutoScalingHandlerTest, URLClassifyProcessorTest, TestFieldSortValues, TestLMJelinekMercerSimilarityFactory, TestConfigSetsAPIZkFailure, OverseerSolrResponseTest, TestElisionMultitermQuery, ConnectionManagerTest, ForceLeaderTest, TestSolrDeletionPolicy1, TestHighlightDedupGrouping, TestSchemaVersionResource, MetricTriggerTest, ConfigureRecoveryStrategyTest, TestMiniSolrCloudClusterSSL, LeaderElectionContextKeyTest, BinaryUpdateRequestHandlerTest, SignatureUpdateProcessorFactoryTest, TestTlogReplayVsRecovery, TestCoreBackup, TestMacroExpander, TestOmitPositions, JvmMetricsTest, EmbeddedSolrNoSerializeTest, TestPayloadScoreQParserPlugin, AutoscalingHistoryHandlerTest, MBeansHandlerTest, TestWithCollection, ManagedSchemaRoundRobinCloudTest, TestFaceting, CoreAdminOperationTest, BasicFunctionalityTest, ConvertedLegacyTest, EchoParamsTest, MinimalSchemaTest, OutputWriterTest, SolrInfoBeanTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestDistributedSearch, TestDocumentBuilder, TestSolrCoreProperties, TestTrie, PathHierarchyTokenizerFactoryTest, ProtectedTermFilterFactoryTest, TestWordDelimiterFilterFactory, TokenizerChainTest, TestLuceneIndexBackCompat, CleanupOldIndexTest, ClusterStateTest, ClusterStateUpdateTest, CollectionPropsTest, DeleteNodeTest, DeleteReplicaTest, LeaderElectionTest, LegacyCloudClusterPropTest, MissingSegmentRecoveryTest, MoveReplicaHDFSFailoverTest, PeerSyncReplicationTest, RollingRestartTest, TestCollectionsAPIViaSolrCloudCluster, TestHdfsCloudBackupRestore, TestLocalFSCloudBackupRestore, TestReplicaProperties, TestRequestStatusCollectionAPI, AutoAddReplicasIntegrationTest, AutoAddReplicasPlanActionTest, ExecutePlanActionTest, HdfsAutoAddReplicasIntegrationTest, HttpTriggerListenerTest, IndexSizeTriggerSizeEstimationTest, NodeAddedTriggerTest, NodeMarkersRegistrationTest, ScheduledTriggerIntegrationTest, SearchRateTriggerTest, TriggerCooldownIntegrationTest, TriggerEventQueueTest, TriggerIntegrationTest, TestSimNodeLostTrigger, TestSimPolicyCloud, TestSimTriggerIntegration, TestSimUtils, CdcrBootstrapTest, CdcrOpsAndBoundariesTest, HDFSCollectionsAPITest, HdfsBasicDistributedZk2Test]
[junit4] Completed [595/910 (2!)] on J1 in 241.30s, 1 test, 1 failure <<< FAILURES!
[...truncated 53117 lines...]