You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/05/25 00:28:53 UTC
[JENKINS] Lucene-Solr-master-Windows (32bit/jdk1.8.0_131) - Build #
6584 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Windows/6584/
Java: 32bit/jdk1.8.0_131 -server -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test
Error Message:
Could not find a healthy node to handle the request.
Stack Trace:
org.apache.solr.common.SolrException: Could not find a healthy node to handle the request.
at __randomizedtesting.SeedInfo.seed([31AC2A64CF30E54C:B9F815BE61CC88B4]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1392)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1139)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1074)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:890)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.tryDelete(ChaosMonkeySafeLeaderWithPullReplicasTest.java:235)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:128)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1713)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:957)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:916)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:802)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:852)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12483 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest
[junit4] 2> Creating dataDir: C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\init-core-data-001
[junit4] 2> 1890347 WARN (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[31AC2A64CF30E54C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=16 numCloses=16
[junit4] 2> 1890347 INFO (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[31AC2A64CF30E54C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields
[junit4] 2> 1890351 INFO (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[31AC2A64CF30E54C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1890351 INFO (SUITE-ChaosMonkeySafeLeaderWithPullReplicasTest-seed#[31AC2A64CF30E54C]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1890351 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ChaosMonkeySafeLeaderWithPullReplicasTest Starting ChaosMonkey test with 1 shards and 2 nodes
[junit4] 2> 1890353 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1890353 INFO (Thread-4505) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1890353 INFO (Thread-4505) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1890356 ERROR (Thread-4505) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1890454 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ZkTestServer start zk server on port:54034
[junit4] 2> 1890466 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1890468 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1890469 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1890473 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1890474 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1890475 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1890476 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1890478 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1890479 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1890480 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1890481 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractZkTestCase put C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\core\src\test-files\solr\collection1\conf\synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1890483 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1890943 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\control-001\cores\collection1
[junit4] 2> 1890945 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1890945 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f4648{/,null,AVAILABLE}
[junit4] 2> 1890946 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@e167d9{HTTP/1.1,[http/1.1]}{127.0.0.1:54053}
[junit4] 2> 1890946 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.Server Started @1893737ms
[junit4] 2> 1890946 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\tempDir-001/control/data, hostContext=/, hostPort=54053, coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\control-001\cores}
[junit4] 2> 1890947 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1890947 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1890947 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1890947 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1890947 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-25T00:01:23.534Z
[junit4] 2> 1890950 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1890950 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\control-001\solr.xml
[junit4] 2> 1890958 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1890962 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
[junit4] 2> 1890981 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1890981 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54053_
[junit4] 2> 1890982 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.c.Overseer Overseer (id=98020260770152452-127.0.0.1:54053_-n_0000000000) starting
[junit4] 2> 1890988 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54053_
[junit4] 2> 1890989 INFO (zkCallback-4192-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1891094 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1891096 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1891096 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1891098 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\control-001\cores
[junit4] 2> 1891098 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54053_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1891099 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1891101 INFO (OverseerStateUpdate-98020260770152452-127.0.0.1:54053_-n_0000000000) [n:127.0.0.1:54053_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1892123 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1892148 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1892222 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1892239 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1892239 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.core_node1' (registry 'solr.core.control_collection.shard1.core_node1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1892239 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1892239 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\control-001\cores\collection1], dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\control-001\cores\collection1\data\]
[junit4] 2> 1892246 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=8.8798828125, floorSegmentMB=1.6572265625, forceMergeDeletesPctAllowed=26.378772813920104, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1892250 WARN (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1892329 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1892329 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1892330 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1892330 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1892332 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=12, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3773882364683808]
[junit4] 2> 1892332 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@768e3a[collection1] main]
[junit4] 2> 1892333 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1892334 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1892334 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1892336 INFO (searcherExecutor-9250-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@768e3a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1892336 INFO (coreLoadExecutor-9249-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ c:control_collection x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1568324174398619648
[junit4] 2> 1892341 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1892341 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1892341 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54053/collection1/
[junit4] 2> 1892342 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1892342 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:54053/collection1/ has no replicas
[junit4] 2> 1892342 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1892344 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54053/collection1/ shard1
[junit4] 2> 1892497 INFO (coreZkRegister-9242-thread-1-processing-n:127.0.0.1:54053_ x:collection1 c:control_collection) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1892624 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1892625 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:54034/solr ready
[junit4] 2> 1892625 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1892625 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Creating collection1 with stateFormat=2
[junit4] 2> 1892993 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-1-001\cores\collection1
[junit4] 2> 1892994 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-1-001 of type PULL
[junit4] 2> 1892995 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1892995 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23d5a2{/,null,AVAILABLE}
[junit4] 2> 1892997 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@18f87eb{HTTP/1.1,[http/1.1]}{127.0.0.1:54083}
[junit4] 2> 1892997 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.Server Started @1895787ms
[junit4] 2> 1892997 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\tempDir-001/jetty1, replicaType=PULL, solrconfig=solrconfig.xml, hostContext=/, hostPort=54083, coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-1-001\cores}
[junit4] 2> 1892997 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1892997 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1892997 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1892997 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1892997 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-25T00:01:25.584Z
[junit4] 2> 1893002 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c3ce867ff0007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1893002 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1893002 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-1-001\solr.xml
[junit4] 2> 1893010 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1893014 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
[junit4] 2> 1893024 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1893026 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1893028 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54083_
[junit4] 2> 1893029 INFO (zkCallback-4192-thread-1-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1893029 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1893030 INFO (zkCallback-4196-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1893140 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1893144 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1893144 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1893147 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-1-001\cores
[junit4] 2> 1893147 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54083_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1893148 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1893149 INFO (OverseerStateUpdate-98020260770152452-127.0.0.1:54053_-n_0000000000) [n:127.0.0.1:54053_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1893251 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1894165 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1894175 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1894226 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1894240 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true
[junit4] 2> 1894240 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.core_node1' (registry 'solr.core.collection1.shard1.core_node1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1894245 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1894245 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-1-001\cores\collection1], dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\..\..\..\..\..\..\..\..\..\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-1-001\cores\collection1\data\]
[junit4] 2> 1894251 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=8.8798828125, floorSegmentMB=1.6572265625, forceMergeDeletesPctAllowed=26.378772813920104, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1894264 WARN (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1894334 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1894334 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1894335 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=12, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3773882364683808]
[junit4] 2> 1894335 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1abc2ff[collection1] main]
[junit4] 2> 1894337 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1894338 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1894338 INFO (coreLoadExecutor-9260-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1894339 INFO (searcherExecutor-9261-thread-1-processing-n:127.0.0.1:54083_ x:collection1 c:collection1) [n:127.0.0.1:54083_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1abc2ff[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1894340 INFO (coreZkRegister-9255-thread-1-processing-n:127.0.0.1:54083_ x:collection1 c:collection1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController collection1 starting background replication from leader
[junit4] 2> 1894340 INFO (coreZkRegister-9255-thread-1-processing-n:127.0.0.1:54083_ x:collection1 c:collection1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 0:0:7
[junit4] 2> 1894343 INFO (coreZkRegister-9255-thread-1-processing-n:127.0.0.1:54083_ x:collection1 c:collection1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 7000ms
[junit4] 2> 1894343 WARN (coreZkRegister-9255-thread-1-processing-n:127.0.0.1:54083_ x:collection1 c:collection1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler SolrCloud is enabled for core collection1 but so is old-style replication. Make sure you intend this behavior, it usually indicates a mis-configuration. Master setting is false and slave setting is true
[junit4] 2> 1895050 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-2-001\cores\collection1
[junit4] 2> 1895052 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-2-001 of type NRT
[junit4] 2> 1895053 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.Server jetty-9.3.14.v20161028
[junit4] 2> 1895053 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1bd4b58{/,null,AVAILABLE}
[junit4] 2> 1895054 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@a368d3{HTTP/1.1,[http/1.1]}{127.0.0.1:54107}
[junit4] 2> 1895054 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.Server Started @1897844ms
[junit4] 2> 1895054 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/, hostPort=54107, coreRootDirectory=C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-2-001\cores}
[junit4] 2> 1895057 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1895057 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.0.0
[junit4] 2> 1895057 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1895057 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1895057 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-05-25T00:01:27.644Z
[junit4] 2> 1895061 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1895061 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-2-001\solr.xml
[junit4] 2> 1895069 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1895073 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54034/solr
[junit4] 2> 1895077 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c3ce867ff000b, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1895083 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1895085 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1895087 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54107_
[junit4] 2> 1895088 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1895088 INFO (zkCallback-4192-thread-2-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1895088 INFO (zkCallback-4196-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1895088 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1895233 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1895236 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1895236 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1895239 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-2-001\cores
[junit4] 2> 1895239 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [n:127.0.0.1:54107_ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 1895240 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1895240 INFO (OverseerStateUpdate-98020260770152452-127.0.0.1:54053_-n_0000000000) [n:127.0.0.1:54053_ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1895343 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1895343 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1896289 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
[junit4] 2> 1896304 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 1897473 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1897513 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection collection1, trusted=true
[junit4] 2> 1897513 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.core_node2' (registry 'solr.core.collection1.shard1.core_node2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1cfe1dd
[junit4] 2> 1897513 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1897513 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-2-001\cores\collection1], dataDir=[C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001\shard-2-001\cores\collection1\data\]
[junit4] 2> 1897523 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=8.8798828125, floorSegmentMB=1.6572265625, forceMergeDeletesPctAllowed=26.378772813920104, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1897540 WARN (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1897711 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1897711 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1897713 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1897714 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1897715 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=12, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3773882364683808]
[junit4] 2> 1897716 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2fe4e1[collection1] main]
[junit4] 2> 1897719 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1897720 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1897720 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000
[junit4] 2> 1897721 INFO (coreLoadExecutor-9272-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1568324180045201408
[junit4] 2> 1897740 INFO (searcherExecutor-9273-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2fe4e1[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1897747 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1897747 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1897747 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:54107/collection1/
[junit4] 2> 1897747 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1897747 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SyncStrategy http://127.0.0.1:54107/collection1/ has no replicas
[junit4] 2> 1897747 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1897749 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1897749 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1897757 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:54107/collection1/ shard1
[junit4] 2> 1897862 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1897863 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1897865 INFO (indexFetcher-9266-thread-1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.h.IndexFetcher Replica core_node2 is leader but it's state is down, skipping replication
[junit4] 2> 1897915 INFO (coreZkRegister-9267-thread-1-processing-n:127.0.0.1:54107_ x:collection1 c:collection1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1897953 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 1897967 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Turning on auto soft commit: 1000
[junit4] 2> 1897994 INFO (qtp20354264-20387) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp= path=/update params={wt=javabin&version=2}{deleteByQuery=*:* (-1568324180306296832)} 0 24
[junit4] 2> 1897995 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection collection1 failed due to (510) org.apache.solr.common.SolrException: Could not find a healthy node to handle the request., retry? 0
[junit4] 2> 1897997 WARN (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Re-trying request to collection(s) collection1 after stale state error from server.
[junit4] 2> 1898005 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection collection1 failed due to (510) org.apache.solr.common.SolrException: Could not find a healthy node to handle the request., retry? 1
[junit4] 2> 1898006 WARN (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Re-trying request to collection(s) collection1 after stale state error from server.
[junit4] 2> 1898006 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection collection1 failed due to (510) org.apache.solr.common.SolrException: Could not find a healthy node to handle the request., retry? 2
[junit4] 2> 1898006 WARN (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Re-trying request to collection(s) collection1 after stale state error from server.
[junit4] 2> 1898007 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection collection1 failed due to (510) org.apache.solr.common.SolrException: Could not find a healthy node to handle the request., retry? 3
[junit4] 2> 1898008 WARN (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Re-trying request to collection(s) collection1 after stale state error from server.
[junit4] 2> 1898009 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection collection1 failed due to (510) org.apache.solr.common.SolrException: Could not find a healthy node to handle the request., retry? 4
[junit4] 2> 1898009 WARN (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Re-trying request to collection(s) collection1 after stale state error from server.
[junit4] 2> 1898012 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.s.i.CloudSolrClient Request to collection collection1 failed due to (510) org.apache.solr.common.SolrException: Could not find a healthy node to handle the request., retry? 5
[junit4] 2> 1898013 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending test
[junit4] 2> 1898013 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c3ce867ff0005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1898013 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 54053
[junit4] 2> 1898013 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=11143607
[junit4] 2> 1898014 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1898015 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 1898017 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 1898019 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1898020 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1898020 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [3])
[junit4] 2> 1898037 INFO (coreCloseExecutor-9278-thread-1) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@1e8e574
[junit4] 2> 1898039 INFO (coreCloseExecutor-9278-thread-1) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.DirectUpdateHandler2 Committing on IndexWriter close.
[junit4] 2> 1898040 INFO (coreCloseExecutor-9278-thread-1) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@94b1c3 commitCommandVersion:0
[junit4] 2> 1898050 INFO (coreCloseExecutor-9278-thread-1) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.control_collection.shard1.core_node1, tag=32040308
[junit4] 2> 1898053 INFO (coreCloseExecutor-9278-thread-1) [n:127.0.0.1:54053_ c:control_collection s:shard1 r:core_node1 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.control_collection.shard1.leader, tag=32040308
[junit4] 2> 1898054 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.Overseer Overseer (id=98020260770152452-127.0.0.1:54053_-n_0000000000) closing
[junit4] 2> 1898054 INFO (OverseerStateUpdate-98020260770152452-127.0.0.1:54053_-n_0000000000) [n:127.0.0.1:54053_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54053_
[junit4] 2> 1898069 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c3ce867ff0004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1898069 WARN (zkCallback-4192-thread-2-processing-n:127.0.0.1:54053_) [n:127.0.0.1:54053_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1898070 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 1898070 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54083_
[junit4] 2> 1898072 INFO (zkCallback-4202-thread-2-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 1898072 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@e167d9{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1898073 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7f4648{/,null,UNAVAILABLE}
[junit4] 2> 1898074 INFO (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.Overseer Overseer (id=98020260770152457-127.0.0.1:54083_-n_0000000001) starting
[junit4] 2> 1898074 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 54083
[junit4] 2> 1898075 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=7992835
[junit4] 2> 1898076 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1898076 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 1898076 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 1898087 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1898088 INFO (coreCloseExecutor-9280-thread-1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@13dae5f
[junit4] 2> 1898119 INFO (coreCloseExecutor-9280-thread-1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.core_node1, tag=20819551
[junit4] 2> 1898121 INFO (coreCloseExecutor-9280-thread-1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=20819551
[junit4] 2> 1898122 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.Overseer Overseer (id=98020260770152457-127.0.0.1:54083_-n_0000000001) closing
[junit4] 2> 1898125 INFO (OverseerStateUpdate-98020260770152457-127.0.0.1:54083_-n_0000000001) [n:127.0.0.1:54083_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54083_
[junit4] 2> 1898130 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15c3ce867ff0009, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1898130 WARN (zkCallback-4202-thread-1-processing-n:127.0.0.1:54083_) [n:127.0.0.1:54083_ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1898137 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:54107_
[junit4] 2> 1898138 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.Overseer Overseer (id=98020260770152460-127.0.0.1:54107_-n_0000000002) starting
[junit4] 2> 1898140 INFO (zkCallback-4208-thread-2-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1898276 INFO (zkCallback-4208-thread-1-processing-n:127.0.0.1:54107_) [n:127.0.0.1:54107_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1898504 ERROR (coreZkRegister-9255-thread-1-processing-n:127.0.0.1:54083_ x:collection1 c:collection1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkController Error getting leader from zk
[junit4] 2> org.apache.solr.common.SolrException: CoreContainer is closed
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1084)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1051)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1007)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:912)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:848)
[junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1898504 ERROR (coreZkRegister-9255-thread-1-processing-n:127.0.0.1:54083_ x:collection1 c:collection1) [n:127.0.0.1:54083_ c:collection1 s:shard1 r:core_node1 x:collection1] o.a.s.c.ZkContainer :org.apache.solr.common.SolrException: Error getting leader from zk for shard shard1
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1040)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:912)
[junit4] 2> at org.apache.solr.cloud.ZkController.register(ZkController.java:848)
[junit4] 2> at org.apache.solr.core.ZkContainer.lambda$registerInZk$0(ZkContainer.java:181)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: CoreContainer is closed
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1084)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeaderProps(ZkController.java:1051)
[junit4] 2> at org.apache.solr.cloud.ZkController.getLeader(ZkController.java:1007)
[junit4] 2> ... 7 more
[junit4] 2>
[junit4] 2> 1898511 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@18f87eb{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1898511 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@23d5a2{/,null,UNAVAILABLE}
[junit4] 2> 1898545 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ChaosMonkey monkey: stop jetty! 54107
[junit4] 2> 1898545 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=21179279
[junit4] 2> 1898549 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1898549 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 1898550 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 1898551 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1898553 INFO (coreCloseExecutor-9286-thread-1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@195a1d7
[junit4] 2> 1898556 INFO (coreCloseExecutor-9286-thread-1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1.shard1.core_node2, tag=26583511
[junit4] 2> 1898557 INFO (coreCloseExecutor-9286-thread-1) [n:127.0.0.1:54107_ c:collection1 s:shard1 r:core_node2 x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.collection1.shard1.leader, tag=26583511
[junit4] 2> 1898561 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.Overseer Overseer (id=98020260770152460-127.0.0.1:54107_-n_0000000002) closing
[junit4] 2> 1898563 INFO (OverseerStateUpdate-98020260770152460-127.0.0.1:54107_-n_0000000002) [n:127.0.0.1:54107_ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:54107_
[junit4] 2> 1898584 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@a368d3{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1898585 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1bd4b58{/,null,UNAVAILABLE}
[junit4] 2> 1898585 ERROR (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1898586 INFO (TEST-ChaosMonkeySafeLeaderWithPullReplicasTest.test-seed#[31AC2A64CF30E54C]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54034 54034
[junit4] 2> 1898586 INFO (Thread-4505) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:54034 54034
[junit4] 2> 1898589 WARN (Thread-4505) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2> 4 /solr/clusterprops.json
[junit4] 2> 3 /solr/security.json
[junit4] 2> 3 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2> 2 /solr/collections/collection1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/live_nodes
[junit4] 2> 4 /solr/collections
[junit4] 2> 3 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=ChaosMonkeySafeLeaderWithPullReplicasTest -Dtests.method=test -Dtests.seed=31AC2A64CF30E54C -Dtests.slow=true -Dtests.locale=sr-Latn -Dtests.timezone=America/Araguaina -Dtests.asserts=true -Dtests.file.encoding=Cp1252
[junit4] ERROR 8.24s J1 | ChaosMonkeySafeLeaderWithPullReplicasTest.test <<<
[junit4] > Throwable #1: org.apache.solr.common.SolrException: Could not find a healthy node to handle the request.
[junit4] > at __randomizedtesting.SeedInfo.seed([31AC2A64CF30E54C:B9F815BE61CC88B4]:0)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1392)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1139)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1244)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1074)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:160)
[junit4] > at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
[junit4] > at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
[junit4] > at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
[junit4] > at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:890)
[junit4] > at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.tryDelete(ChaosMonkeySafeLeaderWithPullReplicasTest.java:235)
[junit4] > at org.apache.solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest.test(ChaosMonkeySafeLeaderWithPullReplicasTest.java:128)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
[junit4] > at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: leaving temporary files on disk at: C:\Users\jenkins\workspace\Lucene-Solr-master-Windows\solr\build\solr-core\test\J1\temp\solr.cloud.ChaosMonkeySafeLeaderWithPullReplicasTest_31AC2A64CF30E54C-001
[junit4] 2> May 25, 2017 12:01:31 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn, timezone=America/Araguaina
[junit4] 2> NOTE: Windows 10 10.0 x86/Oracle Corporation 1.8.0_131 (32-bit)/cpus=3,threads=1,free=291786448,total=519831552
[junit4] 2> NOTE: All tests run in this JVM: [TestDeleteCollectionOnDownNodes, TestSolr4Spatial, TestZkChroot, TestMiniSolrCloudCluster, TestComponentsName, TestSimpleTextCodec, StatsComponentTest, TestAddFieldRealTimeGet, TestPathTrie, CurrencyFieldOpenExchangeTest, MetricsHandlerTest, TestFileDictionaryLookup, TestPayloadCheckQParserPlugin, ConvertedLegacyTest, RankQueryTest, LukeRequestHandlerTest, TestRecovery, CircularListTest, TestCloudPseudoReturnFields, TermVectorComponentDistributedTest, CdcrBootstrapTest, SolrXmlInZkTest, TestSchemaNameResource, TestCoreDiscovery, TestCoreAdminApis, IndexSchemaTest, TestSolrCloudWithDelegationTokens, TestSmileRequest, TestHashQParserPlugin, TestDynamicFieldCollectionResource, TestTrie, SolrCLIZkUtilsTest, TestExceedMaxTermLength, TestConfigSetsAPIZkFailure, TestCrossCoreJoin, TestTrackingShardHandlerFactory, TestLMJelinekMercerSimilarityFactory, TestExpandComponent, TestBadConfig, TestLegacyNumericRangeQueryBuilder, UpdateRequestProcessorFactoryTest, EchoParamsTest, DistributedVersionInfoTest, RulesTest, CollectionReloadTest, TestCollectionAPIs, AsyncCallRequestStatusResponseTest, HdfsChaosMonkeyNothingIsSafeTest, OpenCloseCoreStressTest, CoreMergeIndexesAdminHandlerTest, DistributedFacetPivotWhiteBoxTest, SuggestComponentTest, ZkSolrClientTest, TestMissingGroups, SpatialHeatmapFacetsTest, TestMacroExpander, UtilsToolTest, TestTrieFacet, BasicZkTest, SystemInfoHandlerTest, TestRebalanceLeaders, TestInPlaceUpdatesDistrib, JsonLoaderTest, TermVectorComponentTest, FacetPivotSmallTest, RecoveryAfterSoftCommitTest, TestSolrIndexConfig, TestSolrConfigHandlerCloud, TestSerializedLuceneMatchVersion, PrimUtilsTest, TestEmbeddedSolrServerSchemaAPI, ForceLeaderTest, TestRestManager, BJQParserTest, TestDownShardTolerantSearch, TestSchemalessBufferedUpdates, AddSchemaFieldsUpdateProcessorFactoryTest, TestChildDocTransformer, TestClusterProperties, TestDistributedMissingSort, TestComplexPhraseLeadingWildcard, PreAnalyzedUpdateProcessorTest, RequestHandlersTest, TestSolrCoreProperties, TestSystemCollAutoCreate, CoreAdminCreateDiscoverTest, BasicFunctionalityTest, TestJoin, DOMUtilTest, TestTolerantUpdateProcessorRandomCloud, TolerantUpdateProcessorTest, DebugComponentTest, TestExportWriter, GraphQueryTest, TriLevelCompositeIdRoutingTest, FullSolrCloudDistribCmdsTest, CollectionsAPISolrJTest, TestFieldTypeCollectionResource, SolrGraphiteReporterTest, TestReload, TestMiniSolrCloudClusterSSL, TestOrdValues, DistributedFacetPivotLargeTest, TestHighFrequencyDictionaryFactory, TestBackupRepositoryFactory, TestPullReplica, SpatialRPTFieldTypeTest, TestGraphMLResponseWriter, TestCloudDeleteByQuery, SecurityConfHandlerTest, BlockJoinFacetRandomTest, SSLMigrationTest, TestUninvertingReader, TestUseDocValuesAsStored2, TestFilteredDocIdSet, FieldAnalysisRequestHandlerTest, TestBinaryResponseWriter, TestNonDefinedSimilarityFactory, ChaosMonkeySafeLeaderTest, DirectUpdateHandlerOptimizeTest, TestRecoveryHdfs, JSONWriterTest, ConfigureRecoveryStrategyTest, HLLUtilTest, DeleteInactiveReplicaTest, FileBasedSpellCheckerTest, ConfigSetsAPITest, BigEndianAscendingWordDeserializerTest, TestRangeQuery, TestDocSet, CollectionsAPIDistributedZkTest, TestQueryTypes, UpdateParamsTest, TestSimpleTrackingShardHandler, SharedFSAutoReplicaFailoverUtilsTest, BitVectorTest, SignatureUpdateProcessorFactoryTest, UUIDFieldTest, ClassificationUpdateProcessorIntegrationTest, TestImplicitCoreProperties, OutOfBoxZkACLAndCredentialsProvidersTest, TestFastLRUCache, TestSolrCoreSnapshots, DeleteShardTest, QueryParsingTest, TestNumericTerms32, TestPushWriter, DistanceFunctionTest, TestObjectReleaseTracker, LeaderFailureAfterFreshStartTest, SuggesterTest, TestNestedDocsSort, TestConfig, HighlighterConfigTest, HdfsRecoverLeaseTest, LeaderElectionContextKeyTest, SolrShardReporterTest, TestClassNameShortening, TestReqParamsAPI, TestSolrDeletionPolicy2, TestNoOpRegenerator, TestSearcherReuse, SolrIndexMetricsTest, SolrMetricManagerTest, FullHLLTest, TestManagedSchemaAPI, TestSQLHandler, TestReversedWildcardFilterFactory, SolrCmdDistributorTest, TestDFRSimilarityFactory, TestInitParams, TestRequestForwarding, TestShortCircuitedRequests, SyncSliceTest, TestReplicationHandler, DistributedSpellCheckComponentTest, HardAutoCommitTest, TestCoreContainer, SpellCheckComponentTest, QueryElevationComponentTest, BadIndexSchemaTest, TestSort, TestFiltering, DirectUpdateHandlerTest, HighlighterTest, ShowFileRequestHandlerTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, CoreAdminHandlerTest, SuggesterWFSTTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, QueryEqualityTest, TestRemoteStreaming, TestSolrDeletionPolicy1, CacheHeaderTest, DisMaxRequestHandlerTest, DirectSolrSpellCheckerTest, TestOmitPositions, TestValueSourceCache, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, TestCollationField, TestCSVResponseWriter, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, NumericFieldsTest, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, TestPostingsSolrHighlighter, TestLuceneMatchVersion, TestCharFilters, SynonymTokenizerTest, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, BaseCdcrDistributedZkTest, CdcrRequestHandlerTest, ChaosMonkeySafeLeaderWithPullReplicasTest]
[junit4] Completed [495/713 (1!)] on J1 in 9.28s, 1 test, 1 error <<< FAILURES!
[...truncated 40141 lines...]