You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/01/14 01:34:10 UTC
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk-9) - Build # 398 -
Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/398/
Java: 64bit/jdk-9 -XX:+UseCompressedOops -XX:+UseSerialGC
3 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test
Error Message:
shard1 is not consistent. Got 42 from http://127.0.0.1:57647/collection1_shard1_replica_n21 (previous client) and got 45 from http://127.0.0.1:57652/collection1_shard1_replica_n23
Stack Trace:
java.lang.AssertionError: shard1 is not consistent. Got 42 from http://127.0.0.1:57647/collection1_shard1_replica_n21 (previous client) and got 45 from http://127.0.0.1:57652/collection1_shard1_replica_n23
at __randomizedtesting.SeedInfo.seed([1304F9FFF49A8046:9B50C6255A66EDBE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1328)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1307)
at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:162)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode
Error Message:
did not finish processing changes
Stack Trace:
java.lang.AssertionError: did not finish processing changes
at __randomizedtesting.SeedInfo.seed([1304F9FFF49A8046:B4EBE45C3BD70F5E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:297)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig
Error Message:
expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}>
Stack Trace:
java.lang.AssertionError: expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}>
at __randomizedtesting.SeedInfo.seed([1304F9FFF49A8046:2C8CFA57E3CC70A1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Build Log:
[...truncated 12851 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/init-core-data-001
[junit4] 2> 1299712 WARN (SUITE-ChaosMonkeySafeLeaderTest-seed#[1304F9FFF49A8046]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 1299712 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[1304F9FFF49A8046]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1299713 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[1304F9FFF49A8046]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 1299713 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[1304F9FFF49A8046]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1299713 INFO (SUITE-ChaosMonkeySafeLeaderTest-seed#[1304F9FFF49A8046]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
[junit4] 2> 1299715 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1299715 INFO (Thread-2567) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1299715 INFO (Thread-2567) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1299719 ERROR (Thread-2567) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1299818 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.ZkTestServer start zk server on port:57631
[junit4] 2> 1299839 INFO (zkConnectionManagerCallback-3025-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1299843 INFO (zkConnectionManagerCallback-3027-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1299853 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 1299857 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 1299862 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 1299867 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 1299869 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 1299871 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 1299874 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 1299877 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 1299880 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 1299883 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 1299885 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractZkTestCase put /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 1299888 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 1300363 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1300365 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2593f2c1{/,null,AVAILABLE}
[junit4] 2> 1300366 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@c826ecf{HTTP/1.1,[http/1.1]}{127.0.0.1:57635}
[junit4] 2> 1300366 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server Started @1302003ms
[junit4] 2> 1300366 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/tempDir-001/control/data, hostPort=57635, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/control-001/cores, replicaType=NRT}
[junit4] 2> 1300366 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1300366 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1300366 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1300366 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1300366 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T00:37:06.586766Z
[junit4] 2> 1300368 INFO (zkConnectionManagerCallback-3029-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1300370 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1300370 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/control-001/solr.xml
[junit4] 2> 1300372 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1300372 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1300373 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1300375 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57631/solr
[junit4] 2> 1300376 INFO (zkConnectionManagerCallback-3033-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1300386 INFO (zkConnectionManagerCallback-3035-thread-1-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1300543 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1300544 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57635_
[junit4] 2> 1300546 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.c.Overseer Overseer (id=99345381847072772-127.0.0.1:57635_-n_0000000000) starting
[junit4] 2> 1300558 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57635_
[junit4] 2> 1300563 INFO (zkCallback-3034-thread-1-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1300605 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1300616 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1300616 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1300618 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57635_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/control-001/cores
[junit4] 2> 1300635 INFO (zkConnectionManagerCallback-3041-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1300637 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1300639 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57631/solr ready
[junit4] 2> 1300642 INFO (qtp2140154500-13062) [n:127.0.0.1:57635_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:57635_&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1300647 INFO (OverseerThreadFactory-5916-thread-1-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 1300771 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1300771 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1300877 INFO (zkCallback-3034-thread-1-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1301785 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1301793 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 1301854 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1301883 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 1301883 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1301884 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1301884 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 1301887 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=785764380, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 1301897 WARN (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1301943 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1301943 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1301945 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1301945 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1301946 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.7783203125, noCFSRatio=0.49665265886244403]
[junit4] 2> 1301947 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2742660c[control_collection_shard1_replica_n1] main]
[junit4] 2> 1301948 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1301949 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1301950 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1301950 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589526111893585920
[junit4] 2> 1301951 INFO (searcherExecutor-5919-thread-1-processing-n:127.0.0.1:57635_ x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@2742660c[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1301957 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1301957 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1301957 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57635/control_collection_shard1_replica_n1/
[junit4] 2> 1301957 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1301957 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:57635/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1301957 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1301963 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57635/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 1302075 INFO (zkCallback-3034-thread-2-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1302134 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1302136 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1365
[junit4] 2> 1302139 INFO (qtp2140154500-13062) [n:127.0.0.1:57635_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1302244 INFO (zkCallback-3034-thread-2-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 1302660 INFO (OverseerCollectionConfigSetProcessor-99345381847072772-127.0.0.1:57635_-n_0000000000) [n:127.0.0.1:57635_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1303145 INFO (qtp2140154500-13062) [n:127.0.0.1:57635_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:57635_&wt=javabin&version=2} status=0 QTime=2503
[junit4] 2> 1303153 INFO (zkConnectionManagerCallback-3045-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1303157 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1303160 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:57631/solr ready
[junit4] 2> 1303160 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 1303162 INFO (qtp2140154500-13060) [n:127.0.0.1:57635_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1303167 INFO (OverseerThreadFactory-5916-thread-2-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 1303169 WARN (OverseerThreadFactory-5916-thread-2-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 1303389 INFO (qtp2140154500-13060) [n:127.0.0.1:57635_ ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1303389 INFO (qtp2140154500-13060) [n:127.0.0.1:57635_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=226
[junit4] 2> 1303642 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-1-001 of type NRT
[junit4] 2> 1303642 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1303644 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@21bad0a2{/,null,AVAILABLE}
[junit4] 2> 1303644 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@455219d4{HTTP/1.1,[http/1.1]}{127.0.0.1:57647}
[junit4] 2> 1303644 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server Started @1305281ms
[junit4] 2> 1303644 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/tempDir-001/jetty1, hostPort=57647, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-1-001/cores, replicaType=NRT}
[junit4] 2> 1303645 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1303645 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1303645 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1303645 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1303645 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T00:37:09.865647Z
[junit4] 2> 1303647 INFO (zkConnectionManagerCallback-3047-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1303648 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1303648 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-1-001/solr.xml
[junit4] 2> 1303650 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1303650 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1303651 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1303653 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57631/solr
[junit4] 2> 1303654 INFO (zkConnectionManagerCallback-3051-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1303657 INFO (zkConnectionManagerCallback-3053-thread-1-processing-n:127.0.0.1:57647_) [n:127.0.0.1:57647_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1303666 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57647_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1303674 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57647_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1303678 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57647_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57647_
[junit4] 2> 1303680 INFO (zkCallback-3034-thread-2-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1303680 INFO (zkCallback-3044-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1303680 INFO (zkCallback-3052-thread-1-processing-n:127.0.0.1:57647_) [n:127.0.0.1:57647_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1303722 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57647_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1303735 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57647_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1303735 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57647_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1303737 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57647_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-1-001/cores
[junit4] 2> 1303767 INFO (qtp2140154500-13063) [n:127.0.0.1:57635_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:57647_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1303772 INFO (OverseerCollectionConfigSetProcessor-99345381847072772-127.0.0.1:57635_-n_0000000000) [n:127.0.0.1:57635_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1303772 INFO (OverseerThreadFactory-5916-thread-3-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:57647_ for creating new replica
[junit4] 2> 1303777 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1303777 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1304794 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1304802 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema [collection1_shard1_replica_n21] Schema name=test
[junit4] 2> 1304904 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1304934 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n21' using configuration from collection collection1, trusted=true
[junit4] 2> 1304935 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n21' (registry 'solr.core.collection1.shard1.replica_n21') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1304935 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1304935 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [[collection1_shard1_replica_n21] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-1-001/cores/collection1_shard1_replica_n21], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-1-001/cores/collection1_shard1_replica_n21/data/]
[junit4] 2> 1304941 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=785764380, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 1304962 WARN (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1305028 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1305028 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1305042 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1305042 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1305043 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.7783203125, noCFSRatio=0.49665265886244403]
[junit4] 2> 1305044 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.SolrIndexSearcher Opening [Searcher@48d6bc01[collection1_shard1_replica_n21] main]
[junit4] 2> 1305045 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1305046 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1305047 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1305047 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589526115141025792
[junit4] 2> 1305048 INFO (searcherExecutor-5930-thread-1-processing-n:127.0.0.1:57647_ x:collection1_shard1_replica_n21 s:shard1 c:collection1) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SolrCore [collection1_shard1_replica_n21] Registered new searcher Searcher@48d6bc01[collection1_shard1_replica_n21] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1305056 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1305056 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1305056 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:57647/collection1_shard1_replica_n21/
[junit4] 2> 1305056 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1305056 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.SyncStrategy http://127.0.0.1:57647/collection1_shard1_replica_n21/ has no replicas
[junit4] 2> 1305056 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1305061 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:57647/collection1_shard1_replica_n21/ shard1
[junit4] 2> 1305174 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1305176 INFO (qtp1286951314-13111) [n:127.0.0.1:57647_ c:collection1 s:shard1 x:collection1_shard1_replica_n21] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n21&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1399
[junit4] 2> 1305179 INFO (qtp2140154500-13063) [n:127.0.0.1:57635_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:57647_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1411
[junit4] 2> 1305389 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-2-001 of type NRT
[junit4] 2> 1305389 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1305391 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@603e570b{/,null,AVAILABLE}
[junit4] 2> 1305391 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@75672546{HTTP/1.1,[http/1.1]}{127.0.0.1:57652}
[junit4] 2> 1305391 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server Started @1307028ms
[junit4] 2> 1305391 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/tempDir-001/jetty2, hostPort=57652, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-2-001/cores, replicaType=NRT}
[junit4] 2> 1305391 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1305392 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1305392 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1305392 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1305392 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T00:37:11.612155Z
[junit4] 2> 1305393 INFO (zkConnectionManagerCallback-3057-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1305394 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1305394 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-2-001/solr.xml
[junit4] 2> 1305396 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1305396 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1305396 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1305398 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57631/solr
[junit4] 2> 1305400 INFO (zkConnectionManagerCallback-3061-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1305402 INFO (zkConnectionManagerCallback-3063-thread-1-processing-n:127.0.0.1:57652_) [n:127.0.0.1:57652_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1305412 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57652_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1305416 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57652_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1305419 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57652_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57652_
[junit4] 2> 1305424 INFO (zkCallback-3034-thread-1-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1305424 INFO (zkCallback-3044-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1305424 INFO (zkCallback-3052-thread-1-processing-n:127.0.0.1:57647_) [n:127.0.0.1:57647_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1305424 INFO (zkCallback-3062-thread-1-processing-n:127.0.0.1:57652_) [n:127.0.0.1:57652_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1305464 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57652_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1305476 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57652_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1305476 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57652_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1305478 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57652_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-2-001/cores
[junit4] 2> 1305507 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:57652_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1305510 INFO (OverseerCollectionConfigSetProcessor-99345381847072772-127.0.0.1:57635_-n_0000000000) [n:127.0.0.1:57635_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1305511 INFO (OverseerThreadFactory-5916-thread-4-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:57652_ for creating new replica
[junit4] 2> 1305515 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1305515 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1306534 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1306542 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema [collection1_shard1_replica_n23] Schema name=test
[junit4] 2> 1306618 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 1306637 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica_n23' using configuration from collection collection1, trusted=true
[junit4] 2> 1306638 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard1.replica_n23' (registry 'solr.core.collection1.shard1.replica_n23') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1306638 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1306638 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [[collection1_shard1_replica_n23] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-2-001/cores/collection1_shard1_replica_n23], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-2-001/cores/collection1_shard1_replica_n23/data/]
[junit4] 2> 1306642 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=785764380, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 1306651 WARN (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1306706 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1306707 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1306707 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 1306708 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1306709 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.7783203125, noCFSRatio=0.49665265886244403]
[junit4] 2> 1306709 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.SolrIndexSearcher Opening [Searcher@25c292d9[collection1_shard1_replica_n23] main]
[junit4] 2> 1306711 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 1306711 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 1306711 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1306712 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589526116886904832
[junit4] 2> 1306714 INFO (searcherExecutor-5941-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.SolrCore [collection1_shard1_replica_n23] Registered new searcher Searcher@25c292d9[collection1_shard1_replica_n23] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1306714 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.c.ZkController Core needs to recover:collection1_shard1_replica_n23
[junit4] 2> 1306714 INFO (updateExecutor-3058-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DefaultSolrCoreState Running recovery
[junit4] 2> 1306719 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Starting recovery process. recoveringAfterStartup=true
[junit4] 2> 1306719 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy ###### startupVersions=[[]]
[junit4] 2> 1306719 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Begin buffering updates. core=[collection1_shard1_replica_n23]
[junit4] 2> 1306719 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 1306719 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Publishing state of core [collection1_shard1_replica_n23] as recovering, leader is [http://127.0.0.1:57647/collection1_shard1_replica_n21/] and I am [http://127.0.0.1:57652/collection1_shard1_replica_n23/]
[junit4] 2> 1306720 INFO (qtp1744235816-13142) [n:127.0.0.1:57652_ c:collection1 s:shard1 x:collection1_shard1_replica_n23] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n23&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1205
[junit4] 2> 1306722 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Sending prep recovery command to [http://127.0.0.1:57647]; [WaitForState: action=PREPRECOVERY&core=collection1_shard1_replica_n21&nodeName=127.0.0.1:57652_&coreNodeName=core_node24&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true]
[junit4] 2> 1306723 INFO (qtp1286951314-13112) [n:127.0.0.1:57647_ ] o.a.s.h.a.PrepRecoveryOp Going to wait for coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true, maxTime: 183 s
[junit4] 2> 1306724 INFO (qtp1286951314-13112) [n:127.0.0.1:57647_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:57652_, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_n23","base_url":"http://127.0.0.1:57652","node_name":"127.0.0.1:57652_","state":"down","type":"NRT"}
[junit4] 2> 1306725 INFO (qtp2140154500-13064) [n:127.0.0.1:57635_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:57652_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2} status=0 QTime=1217
[junit4] 2> 1307038 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 3 in directory /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-3-001 of type NRT
[junit4] 2> 1307038 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1307040 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2a3ab11{/,null,AVAILABLE}
[junit4] 2> 1307040 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@278f78f0{HTTP/1.1,[http/1.1]}{127.0.0.1:57659}
[junit4] 2> 1307040 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.e.j.s.Server Started @1308677ms
[junit4] 2> 1307040 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/, solrconfig=solrconfig.xml, solr.data.dir=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/tempDir-001/jetty3, hostPort=57659, coreRootDirectory=/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-3-001/cores}
[junit4] 2> 1307041 ERROR (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1307041 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1307041 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1307041 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1307041 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T00:37:13.261799Z
[junit4] 2> 1307043 INFO (zkConnectionManagerCallback-3068-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1307045 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 1307045 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-3-001/solr.xml
[junit4] 2> 1307047 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 1307047 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 1307048 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 1307050 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:57631/solr
[junit4] 2> 1307050 INFO (zkConnectionManagerCallback-3072-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1307057 INFO (zkConnectionManagerCallback-3074-thread-1-processing-n:127.0.0.1:57659_) [n:127.0.0.1:57659_ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1307069 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57659_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1307072 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57659_ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1307076 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57659_ ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57659_
[junit4] 2> 1307078 INFO (zkCallback-3034-thread-1-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1307078 INFO (zkCallback-3044-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1307078 INFO (zkCallback-3062-thread-1-processing-n:127.0.0.1:57652_) [n:127.0.0.1:57652_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1307078 INFO (zkCallback-3052-thread-1-processing-n:127.0.0.1:57647_) [n:127.0.0.1:57647_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1307079 INFO (zkCallback-3073-thread-1-processing-n:127.0.0.1:57659_) [n:127.0.0.1:57659_ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1307109 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57659_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1307119 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57659_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1307119 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57659_ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74272181
[junit4] 2> 1307120 INFO (TEST-ChaosMonkeySafeLeaderTest.test-seed#[1304F9FFF49A8046]) [n:127.0.0.1:57659_ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest_1304F9FFF49A8046-001/shard-3-001/cores
[junit4] 2> 1307160 INFO (qtp2140154500-13062) [n:127.0.0.1:57635_ ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:57659_&action=ADDREPLICA&collection=collection1&shard=shard1&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1307164 INFO (OverseerCollectionConfigSetProcessor-99345381847072772-127.0.0.1:57635_-n_0000000000) [n:127.0.0.1:57635_ ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1307164 INFO (OverseerThreadFactory-5916-thread-5-processing-n:127.0.0.1:57635_) [n:127.0.0.1:57635_ ] o.a.s.c.AddReplicaCmd Node Identified 127.0.0.1:57659_ for creating new replica
[junit4] 2> 1307168 INFO (qtp1574662657-13177) [n:127.0.0.1:57659_ ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard1_replica_n25&action=CREATE&collection=collection1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1307168 INFO (qtp1574662657-13177) [n:127.0.0.1:57659_ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 1307733 INFO (qtp1286951314-13112) [n:127.0.0.1:57647_ ] o.a.s.h.a.PrepRecoveryOp In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1_shard1_replica_n21, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:57652_, coreNodeName=core_node24, onlyIfActiveCheckResult=false, nodeProps: core_node24:{"core":"collection1_shard1_replica_n23","base_url":"http://127.0.0.1:57652","node_name":"127.0.0.1:57652_","state":"recovering","type":"NRT"}
[junit4] 2> 1307733 INFO (qtp1286951314-13112) [n:127.0.0.1:57647_ ] o.a.s.h.a.PrepRecoveryOp Waited coreNodeName: core_node24, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
[junit4] 2> 1307733 INFO (qtp1286951314-13112) [n:127.0.0.1:57647_ ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={nodeName=127.0.0.1:57652_&onlyIfLeaderActive=true&core=collection1_shard1_replica_n21&coreNodeName=core_node24&action=PREPRECOVERY&checkLive=true&state=recovering&onlyIfLeader=true&wt=javabin&version=2} status=0 QTime=1009
[junit4] 2> 1308190 INFO (qtp1574662657-13177) [n:127.0.0.1:57659_ c:collection1 s:shard1 x:collection1_shard1_replica_n25] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1308214 INFO (qtp1574662657-13177) [n:127.0.0.1:57659_ c:collection1 s:shard1 x:collection1_shard1_replica_n25] o.a.s.s.IndexSchema [collection1_shard1_replica_n25] Schema name=test
[junit4] 2> 1308244 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Attempting to PeerSync from [http://127.0.0.1:57647/collection1_shard1_replica_n21/] - recoveringAfterStartup=[true]
[junit4] 2> 1308244 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.PeerSync PeerSync: core=collection1_shard1_replica_n23 url=http://127.0.0.1:57652 START replicas=[http://127.0.0.1:57647/collection1_shard1_replica_n21/] nUpdates=100
[junit4] 2> 1308260 INFO (qtp1286951314-13114) [n:127.0.0.1:57647_ c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_n21] o.a.s.u.IndexFingerprint IndexFingerprint millis:1.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1308261 INFO (qtp1286951314-13114) [n:127.0.0.1:57647_ c:collection1 s:shard1 r:core_node22 x:collection1_shard1_replica_n21] o.a.s.c.S.Request [collection1_shard1_replica_n21] webapp= path=/get params={distrib=false&qt=/get&getFingerprint=9223372036854775807&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 1308261 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.IndexFingerprint IndexFingerprint millis:0.0 result:{maxVersionSpecified=9223372036854775807, maxVersionEncountered=0, maxInHash=0, versionsHash=0, numVersions=0, numDocs=0, maxDoc=0}
[junit4] 2> 1308262 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.PeerSync We are already in sync. No need to do a PeerSync
[junit4] 2> 1308262 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 1308262 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 1308262 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 1308262 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy PeerSync stage of recovery was successful.
[junit4] 2> 1308262 INFO (recoveryExecutor-3059-thread-1-processing-n:127.0.0.1:57652_ x:collection1_shard1_replica_n23 s:shard1 c:collection1 r:core_node24) [n:127.0.0.1:57652_ c:collection1 s:shard1 r:core_node24 x:collection1_shard1_replica_n23] o.a.s.c.RecoveryStrategy Replaying updates buffered during PeerSync.
[...truncated too long message...]
18400627812291,
[junit4] 2> 218450474007391],
[junit4] 2> "_enqueue_time_":218455802236291,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10161_solr",
[junit4] 2> "127.0.0.1:10175_solr",
[junit4] 2> "127.0.0.1:10165_solr",
[junit4] 2> "127.0.0.1:10171_solr",
[junit4] 2> "127.0.0.1:10160_solr",
[junit4] 2> "127.0.0.1:10168_solr",
[junit4] 2> "127.0.0.1:10169_solr",
[junit4] 2> "127.0.0.1:10163_solr",
[junit4] 2> "127.0.0.1:10173_solr",
[junit4] 2> "127.0.0.1:10166_solr",
[junit4] 2> "127.0.0.1:10162_solr",
[junit4] 2> "127.0.0.1:10170_solr",
[junit4] 2> "127.0.0.1:10174_solr",
[junit4] 2> "127.0.0.1:10159_solr",
[junit4] 2> "127.0.0.1:10167_solr",
[junit4] 2> "127.0.0.1:10164_solr",
[junit4] 2> "127.0.0.1:10172_solr"]}}
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:120)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:296)
[junit4] 2> ... 6 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unexpected exception executing operation: action=MOVEREPLICA&async=node_added_trigger/c6a25c427aa1Tbmj18c3fffopj1sfrzl57l2uu/0&waitForFinalState=true&collection=testNodeAdded&targetNode=127.0.0.1:10170_solr&inPlaceMove=true&replica=core_node98
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:110)
[junit4] 2> ... 7 more
[junit4] 2> Caused by: java.io.IOException: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$226/1524951389@6fd809b7 rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@78a36870[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 401]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:443)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ExecutePlanAction.process(ExecutePlanAction.java:80)
[junit4] 2> ... 7 more
[junit4] 2> Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$226/1524951389@6fd809b7 rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@78a36870[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 401]
[junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2104)
[junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:848)
[junit4] 2> at java.base/java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1397)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:173)
[junit4] 2> at java.base/java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:140)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.submit(SimCloudManager.java:363)
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:440)
[junit4] 2> ... 8 more
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1656, maxMBSortInHeap=6.1881254424451555, sim=RandomSimilarity(queryNorm=true): {}, locale=pa, timezone=US/Mountain
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 9 (64-bit)/cpus=3,threads=1,free=185855376,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [DistributedFacetPivotLongTailTest, TestQueryWrapperFilter, TestPHPSerializedResponseWriter, CloneFieldUpdateProcessorFactoryTest, RegexBytesRefFilterTest, TestXIncludeConfig, TestInPlaceUpdatesDistrib, TestCursorMarkWithoutUniqueKey, SmileWriterTest, TestSlowCompositeReaderWrapper, TestWriterPerf, SolrGraphiteReporterTest, TestObjectReleaseTracker, TestQuerySenderListener, TestStreamBody, CollectionReloadTest, TestMacros, BitVectorTest, HdfsCollectionsAPIDistributedZkTest, NodeMutatorTest, TemplateUpdateProcessorTest, SolrTestCaseJ4Test, TestCloudJSONFacetJoinDomain, TestTlogReplica, TestConfigReload, TestSolrCoreSnapshots, MetricsConfigTest, DirectoryFactoryTest, TestStressRecovery, TestLMDirichletSimilarityFactory, SolrCloudReportersTest, ZkStateReaderTest, WrapperMergePolicyFactoryTest, TestGenericDistributedQueue, TestFieldTypeResource, TestLegacyNumericUtils, DateMathParserTest, SuggesterTest, TestPrepRecovery, TestCSVResponseWriter, DataDrivenBlockJoinTest, FullHLLTest, TestSolr4Spatial, OverseerTest, TestIBSimilarityFactory, CdcrReplicationDistributedZkTest, TestCollectionAPIs, TestTrackingShardHandlerFactory, HdfsThreadLeakTest, TestReversedWildcardFilterFactory, TestConfigSetsAPI, TestJmxIntegration, RulesTest, DistributedVersionInfoTest, TestCloudDeleteByQuery, PathHierarchyTokenizerFactoryTest, SliceStateTest, TestFiltering, TestPerFieldSimilarityWithDefaultOverride, SolrRequestParserTest, TlogReplayBufferedWhileIndexingTest, TestLFUCache, OutputWriterTest, BigEndianAscendingWordDeserializerTest, RequiredFieldsTest, TestRuleBasedAuthorizationPlugin, TestTolerantUpdateProcessorRandomCloud, ResponseLogComponentTest, TestRequestStatusCollectionAPI, MetricsHandlerTest, TestFieldCacheSortRandom, TestFieldCacheWithThreads, HdfsChaosMonkeyNothingIsSafeTest, ResourceLoaderTest, TestTriggerIntegration, TestDistributedSearch, TimeRoutedAliasUpdateProcessorTest, TestUtils, NodeAddedTriggerTest, SimpleMLTQParserTest, TestRangeQuery, DateRangeFieldTest, BlockJoinFacetDistribTest, HdfsDirectoryTest, WordBreakSolrSpellCheckerTest, InfixSuggestersTest, CreateCollectionCleanupTest, HighlighterMaxOffsetTest, ConvertedLegacyTest, TestHalfAndHalfDocValues, SuggesterTSTTest, HttpSolrCallGetCoreTest, TestManagedSchemaAPI, CdcrRequestHandlerTest, ZkCLITest, TestBackupRepositoryFactory, AutoScalingHandlerTest, SpellCheckComponentTest, DirectUpdateHandlerOptimizeTest, CircularListTest, TestStandardQParsers, TestJsonRequest, FieldAnalysisRequestHandlerTest, AddReplicaTest, TestEmbeddedSolrServerAdminHandler, TestManagedResource, TestCloudSchemaless, ReturnFieldsTest, TestScoreJoinQPNoScore, ChaosMonkeySafeLeaderWithPullReplicasTest, DirectUpdateHandlerTest, SearchHandlerTest, TestConfigSetsAPIZkFailure, HighlighterTest, MetricUtilsTest, SolrJmxReporterTest, XsltUpdateRequestHandlerTest, TestRecoveryHdfs, CollectionTooManyReplicasTest, TestPointFields, TestTolerantSearch, TestReloadAndDeleteDocs, RAMDirectoryFactoryTest, DefaultValueUpdateProcessorTest, TestTrieFacet, HdfsRestartWhileUpdatingTest, TestCloudNestedDocsSort, BasicAuthStandaloneTest, ExecutePlanActionTest, DirectSolrConnectionTest, TestCodecSupport, TestAddFieldRealTimeGet, TestDocTermOrds, TestBulkSchemaConcurrent, TestPseudoReturnFields, TestPullReplicaErrorHandling, ChaosMonkeyNothingIsSafeTest, RankQueryTest, TestSearcherReuse, SignatureUpdateProcessorFactoryTest, LeaderInitiatedRecoveryOnShardRestartTest, TestPayloadCheckQParserPlugin, RollingRestartTest, NotRequiredUniqueKeyTest, TestNodeAddedTrigger, CdcrBidirectionalTest, TestRealTimeGet, TestLazyCores, SuggesterFSTTest, HLLUtilTest, TestSolrCoreParser, TestSolrCloudWithDelegationTokens, ZkControllerTest, TestRawResponseWriter, TestSweetSpotSimilarityFactory, DistributedDebugComponentTest, ClassificationUpdateProcessorIntegrationTest, TestDynamicFieldResource, TestOmitPositions, DistributedQueryElevationComponentTest, TestFileDictionaryLookup, NumberUtilsTest, BlockDirectoryTest, LeaderElectionTest, SearchRateTriggerTest, BinaryUpdateRequestHandlerTest, CursorPagingTest, BadCopyFieldTest, LeaderFailureAfterFreshStartTest, V2StandaloneTest, TestRandomRequestDistribution, ConfigSetsAPITest, ChangedSchemaMergeTest, CoreAdminCreateDiscoverTest, ConfigureRecoveryStrategyTest, TestJsonFacetRefinement, UUIDFieldTest, CleanupOldIndexTest, TestLegacyField, TestDownShardTolerantSearch, TestSchemaResource, TestSystemCollAutoCreate, TestClusterStateProvider, UpdateLogTest, DistributedSpellCheckComponentTest, TestFuzzyAnalyzedSuggestions, RegexBoostProcessorTest, SolrMetricManagerTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, SyncSliceTest, ShardRoutingTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, ClusterStateUpdateTest, TestRandomFaceting, TestRandomDVFaceting, ShardRoutingCustomTest, TestRecovery, TermVectorComponentDistributedTest, TestJoin, TestStressVersions, TestCoreContainer, StatsComponentTest, SoftAutoCommitTest, AssignTest, CustomCollectionTest, DeleteInactiveReplicaTest, DeleteReplicaTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, DocValuesNotIndexedTest, ForceLeaderTest, MultiThreadedOCPTest, PeerSyncReplicationTest, ReplaceNodeTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudInspectUtil, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCloudSearcherWarming, TestCollectionAPI, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIExclusivity, TestCryptoKeys, TestDeleteCollectionOnDownNodes, TestHdfsCloudBackupRestore, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestLocalFSCloudBackupRestore, TestLockTree, TestMiniSolrCloudClusterSSL, TestPullReplica, TestSolrCloudWithKerberosAlt, TestSolrCloudWithSecureImpersonation, TestStressInPlaceUpdates, AutoAddReplicasIntegrationTest, HttpTriggerListenerTest, NodeLostTriggerTest, TestComputePlanAction, TestDistribStateManager, TestLargeCluster]
[junit4] Completed [590/766 (3!)] on J0 in 36.20s, 4 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 41458 lines...]
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 401 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/401/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
6 tests failed.
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Error from server at http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6: ClusterState says we are the leader (http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6), but locally we don't think so. Request came from null
Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: Error from server at http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6: ClusterState says we are the leader (http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6), but locally we don't think so. Request came from null
at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:D5AE5A5CFCFAE205]:0)
at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:550)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1013)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:946)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:460)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6: ClusterState says we are the leader (http://127.0.0.1:60463/solr/awhollynewcollection_0_shard4_replica_n6), but locally we don't think so. Request came from null
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:527)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
... 1 more
FAILED: org.apache.solr.cloud.TestPullReplica.testRealTimeGet
Error Message:
Unexpected replica count null Live Nodes: [127.0.0.1:59416_solr, 127.0.0.1:59417_solr] Last available state: null
Stack Trace:
java.lang.AssertionError: Unexpected replica count
null
Live Nodes: [127.0.0.1:59416_solr, 127.0.0.1:59417_solr]
Last available state: null
at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:C5B6DBEB58736359]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.SolrCloudTestCase.waitForState(SolrCloudTestCase.java:269)
at org.apache.solr.cloud.TestPullReplica.testRealTimeGet(TestPullReplica.java:329)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete {seed=[9DDB2EE8FAC9CD90:86CBDFA40C3935A0]}
Error Message:
Could not find collection : pull_replica_test_create_delete
Stack Trace:
org.apache.solr.common.SolrException: Could not find collection : pull_replica_test_create_delete
at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:86CBDFA40C3935A0]:0)
at org.apache.solr.common.cloud.ClusterState.getCollection(ClusterState.java:118)
at org.apache.solr.cloud.SolrCloudTestCase.getCollectionState(SolrCloudTestCase.java:247)
at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:162)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.TestPullReplica.testCreateDelete {seed=[9DDB2EE8FAC9CD90:329D635838FBFE8C]}
Error Message:
Too many open files
Stack Trace:
java.net.SocketException: Too many open files
at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90:329D635838FBFE8C]:0)
at java.net.Socket.createImpl(Socket.java:460)
at java.net.Socket.getImpl(Socket.java:520)
at java.net.Socket.setSoTimeout(Socket.java:1141)
at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:120)
at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:359)
at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:381)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:237)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.cloud.TestPullReplica.testCreateDelete(TestPullReplica.java:156)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.search.join.TestCloudNestedDocsSort
Error Message:
Could not find collection:collection1
Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
at org.apache.solr.search.join.TestCloudNestedDocsSort.setupCluster(TestCloudNestedDocsSort.java:75)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.servlet.HttpSolrCallGetCoreTest
Error Message:
Could not find collection:collection1
Stack Trace:
java.lang.AssertionError: Could not find collection:collection1
at __randomizedtesting.SeedInfo.seed([9DDB2EE8FAC9CD90]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:155)
at org.apache.solr.servlet.HttpSolrCallGetCoreTest.setupCluster(HttpSolrCallGetCoreTest.java:52)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13207 lines...]
[junit4] Suite: org.apache.solr.cloud.TestPullReplica
[junit4] 2> 3112347 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/init-core-data-001
[junit4] 2> 3112348 WARN (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 3112348 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3112355 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 3112356 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001
[junit4] 2> 3112356 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3112356 INFO (Thread-3126) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3112356 INFO (Thread-3126) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3112359 ERROR (Thread-3126) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3112465 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:59414
[junit4] 2> 3112481 INFO (zkConnectionManagerCallback-3595-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112490 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3112490 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4acb9b0f{/solr,null,AVAILABLE}
[junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@642dbe{/solr,null,AVAILABLE}
[junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6c230b37{HTTP/1.1,[http/1.1]}{127.0.0.1:59417}
[junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.e.j.s.Server Started @3116315ms
[junit4] 2> 3112493 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59417}
[junit4] 2> 3112495 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3f540ccf{HTTP/1.1,[http/1.1]}{127.0.0.1:59416}
[junit4] 2> 3112495 INFO (jetty-launcher-3592-thread-1) [ ] o.e.j.s.Server Started @3116317ms
[junit4] 2> 3112495 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59416}
[junit4] 2> 3112495 ERROR (jetty-launcher-3592-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3112495 ERROR (jetty-launcher-3592-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T15:47:05.222Z
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3112496 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T15:47:05.222Z
[junit4] 2> 3112498 INFO (zkConnectionManagerCallback-3597-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112498 INFO (zkConnectionManagerCallback-3599-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112499 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3112499 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3112528 INFO (jetty-launcher-3592-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59414/solr
[junit4] 2> 3112528 INFO (jetty-launcher-3592-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:59414/solr
[junit4] 2> 3112530 INFO (zkConnectionManagerCallback-3605-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112530 INFO (zkConnectionManagerCallback-3607-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112533 INFO (zkConnectionManagerCallback-3610-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112533 INFO (zkConnectionManagerCallback-3611-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112711 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3112712 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3112712 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59416_solr
[junit4] 2> 3112713 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59417_solr
[junit4] 2> 3112714 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.Overseer Overseer (id=99354622367105029-127.0.0.1:59416_solr-n_0000000000) starting
[junit4] 2> 3112717 INFO (zkCallback-3608-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3112719 INFO (zkCallback-3609-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3112728 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59416_solr
[junit4] 2> 3112730 INFO (zkCallback-3609-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3112730 INFO (zkCallback-3608-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3112818 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3112824 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3112824 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3112825 INFO (jetty-launcher-3592-thread-1) [n:127.0.0.1:59416_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/.
[junit4] 2> 3112861 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3112867 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3112867 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3112868 INFO (jetty-launcher-3592-thread-2) [n:127.0.0.1:59417_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/.
[junit4] 2> 3112891 INFO (zkConnectionManagerCallback-3617-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112894 INFO (zkConnectionManagerCallback-3621-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3112896 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3112897 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:59414/solr ready
[junit4] 2> 3112903 INFO (SUITE-TestPullReplica-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.TestPullReplica Using legacyCloud?: false
[junit4] 2> 3112907 INFO (qtp1042818031-17484) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :clusterprop with params val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3112908 INFO (qtp1042818031-17484) [n:127.0.0.1:59416_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={val=false&name=legacyCloud&action=CLUSTERPROP&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 3113016 INFO (TEST-TestPullReplica.testRealTimeGet-seed#[9DDB2EE8FAC9CD90]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRealTimeGet
[junit4] 2> 3113018 INFO (qtp1042818031-17486) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=2&replicationFactor=2&collection.configName=conf&maxShardsPerNode=100&name=pull_replica_test_real_time_get&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3113021 INFO (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.CreateCollectionCmd Create collection pull_replica_test_real_time_get
[junit4] 2> 3113022 WARN (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 4 on collection pull_replica_test_real_time_get is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 3113136 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:59416/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3113140 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:59417/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3113144 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_p4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:59416/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3113148 INFO (OverseerStateUpdate-99354622367105029-127.0.0.1:59416_solr-n_0000000000) [n:127.0.0.1:59416_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"pull_replica_test_real_time_get",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"pull_replica_test_real_time_get_shard1_replica_p6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:59417/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3113357 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p4&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 3113357 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n2&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3113358 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3113358 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3113358 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p6&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 3113358 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3113472 INFO (zkCallback-3608-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3113472 INFO (zkCallback-3609-thread-1-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3113472 INFO (zkCallback-3608-thread-2-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3113473 INFO (zkCallback-3609-thread-2-processing-n:127.0.0.1:59417_solr) [n:127.0.0.1:59417_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/pull_replica_test_real_time_get/state.json] for collection [pull_replica_test_real_time_get] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3114390 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 3114391 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 3114403 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 3114403 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 3114408 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_p6] Schema name=minimal
[junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_p6' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 3114408 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_p4] Schema name=minimal
[junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.core.pull_replica_test_real_time_get.shard1.replica_p6' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_p6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3114412 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_p6] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/pull_replica_test_real_time_get_shard1_replica_p6], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/./pull_replica_test_real_time_get_shard1_replica_p6/data/]
[junit4] 2> 3114413 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3114413 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_p4' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 3114414 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.core.pull_replica_test_real_time_get.shard1.replica_p4' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3114414 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3114414 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_p4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/pull_replica_test_real_time_get_shard1_replica_p4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/./pull_replica_test_real_time_get_shard1_replica_p4/data/]
[junit4] 2> 3114416 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_n2] Schema name=minimal
[junit4] 2> 3114417 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.IndexSchema [pull_replica_test_real_time_get_shard1_replica_n1] Schema name=minimal
[junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_n2' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59417.solr.core.pull_replica_test_real_time_get.shard1.replica_n2' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3114428 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/pull_replica_test_real_time_get_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node2/./pull_replica_test_real_time_get_shard1_replica_n2/data/]
[junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'pull_replica_test_real_time_get_shard1_replica_n1' using configuration from collection pull_replica_test_real_time_get, trusted=true
[junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59416.solr.core.pull_replica_test_real_time_get.shard1.replica_n1' (registry 'solr.core.pull_replica_test_real_time_get.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@10f4cc7d
[junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3114429 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore [[pull_replica_test_real_time_get_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/pull_replica_test_real_time_get_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.TestPullReplica_9DDB2EE8FAC9CD90-001/tempDir-001/node1/./pull_replica_test_real_time_get_shard1_replica_n1/data/]
[junit4] 2> 3114506 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@190e5b52
[junit4] 2> 3114506 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_n2, tag=420371282
[junit4] 2> 3114509 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5a7480b5: rootName = solr_59417, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@7b47a79d
[junit4] 2> 3114509 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3dfdb0dc
[junit4] 2> 3114509 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_n1, tag=1040036060
[junit4] 2> 3114509 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@48f77088: rootName = solr_59416, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@41d2887a
[junit4] 2> 3114510 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p4] CLOSING SolrCore org.apache.solr.core.SolrCore@35e4ce9f
[junit4] 2> 3114512 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.c.SolrCore [pull_replica_test_real_time_get_shard1_replica_p6] CLOSING SolrCore org.apache.solr.core.SolrCore@7fb6c75f
[junit4] 2> 3114512 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=420371282
[junit4] 2> 3114513 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_p6, tag=2142685023
[junit4] 2> 3114513 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7296ee99: rootName = solr_59417, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_p6, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_p6 / com.codahale.metrics.MetricRegistry@7d7c988e
[junit4] 2> 3114515 ERROR (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n2': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n2] Caused by: invalid API spec: apispec/core.SchemaEdit.json
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] 2> at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_n2]
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1055)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950)
[junit4] 2> ... 35 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/core.SchemaEdit.json
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1008)
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039)
[junit4] 2> ... 36 more
[junit4] 2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/core.SchemaEdit.json
[junit4] 2> at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
[junit4] 2> at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:427)
[junit4] 2> at org.apache.solr.api.Api.getSpec(Api.java:65)
[junit4] 2> at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
[junit4] 2> at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
[junit4] 2> at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
[junit4] 2> at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:966)
[junit4] 2> ... 38 more
[junit4] 2>
[junit4] 2> 3114515 INFO (qtp1560461278-17482) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node5 x:pull_replica_test_real_time_get_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n2&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=400 QTime=1157
[junit4] 2> 3114516 ERROR (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:59417/solr
[junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59417/solr: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n2': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n2] Caused by: invalid API spec: apispec/core.SchemaEdit.json
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3114521 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=1040036060
[junit4] 2> 3114523 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.pull_replica_test_real_time_get.shard1.replica_p4, tag=904187551
[junit4] 2> 3114524 ERROR (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n1': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n1] Caused by: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_n1]
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1055)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950)
[junit4] 2> ... 34 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1008)
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039)
[junit4] 2> ... 35 more
[junit4] 2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
[junit4] 2> at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:427)
[junit4] 2> at org.apache.solr.api.Api.getSpec(Api.java:65)
[junit4] 2> at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
[junit4] 2> at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
[junit4] 2> at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
[junit4] 2> at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:966)
[junit4] 2> ... 37 more
[junit4] 2>
[junit4] 2> 3114524 INFO (qtp1042818031-17488) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node3 x:pull_replica_test_real_time_get_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_n1&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=400 QTime=1166
[junit4] 2> 3114525 ERROR (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:59416/solr
[junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59416/solr: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_n1': Unable to create core [pull_replica_test_real_time_get_shard1_replica_n1] Caused by: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3114528 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2c3163f3: rootName = solr_59416, domain = solr.core.pull_replica_test_real_time_get.shard1.replica_p4, service url = null, agent id = null] for registry solr.core.pull_replica_test_real_time_get.shard1.replica_p4 / com.codahale.metrics.MetricRegistry@7f25f41e
[junit4] 2> 3114528 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=2142685023
[junit4] 2> 3114529 ERROR (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_p6': Unable to create core [pull_replica_test_real_time_get_shard1_replica_p6] Caused by: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_p6]
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1055)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950)
[junit4] 2> ... 34 more
[junit4] 2> Caused by: org.apache.solr.common.SolrException: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:1008)
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:863)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1039)
[junit4] 2> ... 35 more
[junit4] 2> Caused by: java.lang.RuntimeException: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.common.util.ValidatingJsonMap.parse(ValidatingJsonMap.java:318)
[junit4] 2> at org.apache.solr.common.util.Utils.lambda$getSpec$0(Utils.java:427)
[junit4] 2> at org.apache.solr.api.Api.getSpec(Api.java:65)
[junit4] 2> at org.apache.solr.api.ApiBag.register(ApiBag.java:73)
[junit4] 2> at org.apache.solr.core.PluginBag.put(PluginBag.java:217)
[junit4] 2> at org.apache.solr.core.PluginBag.init(PluginBag.java:274)
[junit4] 2> at org.apache.solr.core.RequestHandlers.initHandlersFromConfig(RequestHandlers.java:130)
[junit4] 2> at org.apache.solr.core.SolrCore.<init>(SolrCore.java:966)
[junit4] 2> ... 37 more
[junit4] 2>
[junit4] 2> 3114530 INFO (qtp1560461278-17485) [n:127.0.0.1:59417_solr c:pull_replica_test_real_time_get s:shard1 r:core_node8 x:pull_replica_test_real_time_get_shard1_replica_p6] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=pull_replica_test_real_time_get_shard1_replica_p6&action=CREATE&numShards=1&collection=pull_replica_test_real_time_get&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=400 QTime=1171
[junit4] 2> 3114530 ERROR (OverseerThreadFactory-6833-thread-1-processing-n:127.0.0.1:59416_solr) [n:127.0.0.1:59416_solr ] o.a.s.c.OverseerCollectionMessageHandler Error from shard: http://127.0.0.1:59417/solr
[junit4] 2> org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:59417/solr: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_p6': Unable to create core [pull_replica_test_real_time_get_shard1_replica_p6] Caused by: invalid API spec: apispec/emptySpec.json
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
[junit4] 2> at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
[junit4] 2> at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
[junit4] 2> at org.apache.solr.handler.component.HttpShardHandler.lambda$submit$0(HttpShardHandler.java:172)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3114533 INFO (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.pull_replica_test_real_time_get.shard1.leader, tag=904187551
[junit4] 2> 3114537 ERROR (qtp1042818031-17487) [n:127.0.0.1:59416_solr c:pull_replica_test_real_time_get s:shard1 r:core_node7 x:pull_replica_test_real_time_get_shard1_replica_p4] o.a.s.h.RequestHandlerBase org.apache.solr.common.SolrException: Error CREATEing SolrCore 'pull_replica_test_real_time_get_shard1_replica_p4': Unable to create core [pull_replica_test_real_time_get_shard1_replica_p4] Caused by: invalid API spec: apispec/core.SchemaEdit.json
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:989)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.solr.common.SolrException: Unable to create core [pull_replica_test_real_time_get_shard1_replica_p4]
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:105
[...truncated too long message...]
[junit4] 2> 3644771 INFO (jetty-closer-4002-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3644772 INFO (jetty-closer-4002-thread-4) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:60464_solr
[junit4] 2> 3644773 INFO (zkCallback-4029-thread-3-processing-n:127.0.0.1:60464_solr) [n:127.0.0.1:60464_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2)
[junit4] 2> 3644773 INFO (zkCallback-4070-thread-2-processing-n:127.0.0.1:60463_solr) [n:127.0.0.1:60463_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2)
[junit4] 2> 3644773 INFO (zkCallback-4031-thread-7-processing-n:127.0.0.1:60465_solr) [n:127.0.0.1:60465_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2)
[junit4] 2> 3644773 INFO (zkCallback-4035-thread-4-processing-n:127.0.0.1:60466_solr) [n:127.0.0.1:60466_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (2)
[junit4] 2> 3644776 INFO (jetty-closer-4002-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@73e6ceb4{/solr,null,UNAVAILABLE}
[junit4] 2> 3644778 INFO (zkCallback-4070-thread-6-processing-n:127.0.0.1:60463_solr) [n:127.0.0.1:60463_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 3644779 INFO (zkCallback-4029-thread-1-processing-n:127.0.0.1:60464_solr) [n:127.0.0.1:60464_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 3644780 INFO (jetty-closer-4002-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@30c66fd8{/solr,null,UNAVAILABLE}
[junit4] 2> 3644783 INFO (jetty-closer-4002-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:60463_solr
[junit4] 2> 3644783 INFO (jetty-closer-4002-thread-4) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3644784 INFO (zkCallback-4070-thread-6-processing-n:127.0.0.1:60463_solr) [n:127.0.0.1:60463_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 3644785 INFO (zkCallback-4029-thread-1-processing-n:127.0.0.1:60464_solr) [n:127.0.0.1:60464_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 3644786 INFO (jetty-closer-4002-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3644786 INFO (jetty-closer-4002-thread-4) [ ] o.a.s.c.Overseer Overseer (id=99354647925817353-127.0.0.1:60464_solr-n_0000000000) closing
[junit4] 2> 3644787 INFO (OverseerStateUpdate-99354647925817353-127.0.0.1:60464_solr-n_0000000000) [n:127.0.0.1:60464_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:60464_solr
[junit4] 2> 3644789 INFO (jetty-closer-4002-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3d68c24c{/solr,null,UNAVAILABLE}
[junit4] 2> 3644789 WARN (OverseerAutoScalingTriggerThread-99354647925817353-127.0.0.1:60464_solr-n_0000000000) [n:127.0.0.1:60464_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 3644798 INFO (jetty-closer-4002-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6c85c338{/solr,null,UNAVAILABLE}
[junit4] 2> 3644799 ERROR (SUITE-CollectionsAPIDistributedZkTest-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3644800 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[9DDB2EE8FAC9CD90]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60461 60461
[junit4] 2> 3644803 INFO (Thread-3419) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60461 60461
[junit4] 2> 3645000 WARN (Thread-3419) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 9 /solr/aliases.json
[junit4] 2> 9 /solr/clusterprops.json
[junit4] 2> 7 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 15 /solr/collections/acollectionafterbaddelete/state.json
[junit4] 2> 13 /solr/collections/onlyinzk/state.json
[junit4] 2> 10 /solr/collections/awhollynewcollection_0/state.json
[junit4] 2> 9 /solr/clusterstate.json
[junit4] 2> 5 /solr/collections/halfcollectionblocker/state.json
[junit4] 2> 5 /solr/collections/halfcollection/state.json
[junit4] 2> 5 /solr/collections/created_and_deleted/state.json
[junit4] 2> 5 /solr/collections/withconfigset2/state.json
[junit4] 2> 5 /solr/collections/halfcollectionblocker2/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 324 /solr/collections
[junit4] 2> 9 /solr/live_nodes
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_9DDB2EE8FAC9CD90-001
[junit4] 2> Jan 15, 2018 3:55:57 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=sv-SE, timezone=America/Atka
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=137806224,total=467017728
[junit4] 2> NOTE: All tests run in this JVM: [TestFastWriter, TestExactSharedStatsCache, TestConfig, V2ApiIntegrationTest, CoreAdminRequestStatusTest, TestPostingsSolrHighlighter, DeleteLastCustomShardedReplicaTest, CdcrReplicationHandlerTest, TestLegacyTerms, RecoveryAfterSoftCommitTest, SystemInfoHandlerTest, ComputePlanActionTest, ZkSolrClientTest, TestComplexPhraseLeadingWildcard, ZkNodePropsTest, TestExecutePlanAction, TestFieldCacheWithThreads, TestValueSourceCache, TestSchemaManager, FileUtilsTest, TestExtendedDismaxParser, HdfsNNFailoverTest, TestQueryWrapperFilter, TestReload, TestBadConfig, DebugComponentTest, BasicDistributedZk2Test, TestInfoStreamLogging, QueryEqualityTest, TestSuggestSpellingConverter, TestInPlaceUpdatesDistrib, TestLargeCluster, DeleteReplicaTest, TestNodeLostTrigger, MetricUtilsTest, TestRuleBasedAuthorizationPlugin, TestBlobHandler, SystemLogListenerTest, TestCrossCoreJoin, TestStressLucene, RequestLoggingTest, TestJettySolrRunner, TestRequestForwarding, FieldMutatingUpdateProcessorTest, PingRequestHandlerTest, TestCollectionsAPIViaSolrCloudCluster, TermVectorComponentDistributedTest, TestSort, TestDocTermOrds, TestBM25SimilarityFactory, TestElisionMultitermQuery, DistributedQueryComponentOptimizationTest, TestUtilizeNode, SearchHandlerTest, SpatialHeatmapFacetsTest, TestCloudManagedSchema, TestDefaultStatsCache, DocValuesMissingTest, BasicFunctionalityTest, HdfsAutoAddReplicasIntegrationTest, TestSolrConfigHandlerConcurrent, TestStressReorder, SolrCoreTest, TestConfigSetsAPIExclusivity, TestDFISimilarityFactory, TestStressCloudBlindAtomicUpdates, UninvertDocValuesMergePolicyTest, OverseerCollectionConfigSetProcessorTest, TestPivotHelperCode, UpdateParamsTest, ZkFailoverTest, DistributedFacetPivotLargeTest, TestMinMaxOnMultiValuedField, TestSimpleTrackingShardHandler, ChaosMonkeyNothingIsSafeWithPullReplicasTest, MultiThreadedOCPTest, TestBulkSchemaAPI, TestFieldTypeCollectionResource, TestGroupingSearch, TestClusterStateMutator, TestAnalyzeInfixSuggestions, TestHighFrequencyDictionaryFactory, TestQueryUtils, SegmentsInfoRequestHandlerTest, SSLMigrationTest, TestTrie, TestNRTOpen, HdfsTlogReplayBufferedWhileIndexingTest, TestRTGBase, LeaderElectionIntegrationTest, TestMacroExpander, TriggerIntegrationTest, ConjunctionSolrSpellCheckerTest, SolrIndexConfigTest, TestRebalanceLeaders, TestZkChroot, TermVectorComponentTest, TestJsonRequest, SolrPluginUtilsTest, CollectionsAPIAsyncDistributedZkTest, PluginInfoTest, TestConfigOverlay, AliasIntegrationTest, TestExportWriter, AnalysisAfterCoreReloadTest, LeaderInitiatedRecoveryOnCommitTest, TestUseDocValuesAsStored, TestLeaderInitiatedRecoveryThread, LoggingHandlerTest, SuggestComponentTest, CurrencyRangeFacetCloudTest, InfoHandlerTest, TestNumericRangeQuery32, ZkStateWriterTest, TestLuceneIndexBackCompat, TestSolrCloudWithKerberosAlt, TestPerFieldSimilarity, TestCloudPivotFacet, TestSolrQueryParser, TestApiFramework, SubstringBytesRefFilterTest, TestTrieFacet, OpenCloseCoreStressTest, TestMergePolicyConfig, SolrSlf4jReporterTest, TestSSLRandomization, SpatialRPTFieldTypeTest, TestShortCircuitedRequests, TestSQLHandlerNonCloud, SimplePostToolTest, BaseCdcrDistributedZkTest, DistributedMLTComponentTest, TestSubQueryTransformerDistrib, CdcrBootstrapTest, BooleanFieldTest, TestDocSet, AssignTest, FullSolrCloudDistribCmdsTest, DocValuesNotIndexedTest, BlobRepositoryCloudTest, FacetPivotSmallTest, TestAddFieldRealTimeGet, DistributedQueueTest, TestSolrDeletionPolicy1, AddSchemaFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, TestPullReplica, TemplateUpdateProcessorTest, HttpSolrCallGetCoreTest, UUIDUpdateProcessorFallbackTest, TestCloudNestedDocsSort, StatelessScriptUpdateProcessorFactoryTest, TestComponentsName, TestCharFilters, TestDynamicLoading, TestFaceting, SynonymTokenizerTest, CurrencyFieldTypeTest, TestFastOutputStream, CollectionsAPIDistributedZkTest]
[junit4] Completed [499/767 (4!)] on J1 in 143.19s, 19 tests, 1 error <<< FAILURES!
[...truncated 49042 lines...]
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk-9) - Build # 400 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/400/
Java: 64bit/jdk-9 -XX:-UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig
Error Message:
expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}>
Stack Trace:
java.lang.AssertionError: expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}>
at __randomizedtesting.SeedInfo.seed([819A134696D623FA:BE1210EE8180D31D]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:564)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:844)
Build Log:
[...truncated 12161 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider
[junit4] 2> 581925 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/init-core-data-001
[junit4] 2> 581925 WARN (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 581925 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 581926 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason="", value=0.0/0.0, ssl=0.0/0.0, clientAuth=0.0/0.0) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 581926 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.c.a.s.TestClusterStateProvider ####### Using simulated components? true
[junit4] 2> 581926 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001
[junit4] 2> 581926 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 581927 INFO (Thread-889) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 581927 INFO (Thread-889) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 581929 ERROR (Thread-889) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 582031 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:58203
[junit4] 2> 582049 INFO (zkConnectionManagerCallback-1160-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582054 INFO (jetty-launcher-1157-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 582054 INFO (jetty-launcher-1157-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 582054 INFO (jetty-launcher-1157-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 582057 INFO (jetty-launcher-1157-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@23d5e0e7{/solr,null,AVAILABLE}
[junit4] 2> 582058 INFO (jetty-launcher-1157-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@32cf1c5a{/solr,null,AVAILABLE}
[junit4] 2> 582058 INFO (jetty-launcher-1157-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@53816dc3{/solr,null,AVAILABLE}
[junit4] 2> 582059 INFO (jetty-launcher-1157-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6ebda1a5{HTTP/1.1,[http/1.1]}{127.0.0.1:58205}
[junit4] 2> 582059 INFO (jetty-launcher-1157-thread-3) [ ] o.e.j.s.Server Started @583799ms
[junit4] 2> 582059 INFO (jetty-launcher-1157-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@66a49ca4{HTTP/1.1,[http/1.1]}{127.0.0.1:58207}
[junit4] 2> 582059 INFO (jetty-launcher-1157-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58205}
[junit4] 2> 582059 INFO (jetty-launcher-1157-thread-2) [ ] o.e.j.s.Server Started @583799ms
[junit4] 2> 582059 INFO (jetty-launcher-1157-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58207}
[junit4] 2> 582059 ERROR (jetty-launcher-1157-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 582059 ERROR (jetty-launcher-1157-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T02:57:18.965397Z
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 582060 INFO (jetty-launcher-1157-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T02:57:18.965602Z
[junit4] 2> 582061 INFO (jetty-launcher-1157-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@29c5aa16{HTTP/1.1,[http/1.1]}{127.0.0.1:58206}
[junit4] 2> 582061 INFO (jetty-launcher-1157-thread-1) [ ] o.e.j.s.Server Started @583802ms
[junit4] 2> 582061 INFO (jetty-launcher-1157-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58206}
[junit4] 2> 582063 ERROR (jetty-launcher-1157-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 582063 INFO (zkConnectionManagerCallback-1162-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582063 INFO (jetty-launcher-1157-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 582063 INFO (jetty-launcher-1157-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 582063 INFO (jetty-launcher-1157-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 582063 INFO (jetty-launcher-1157-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T02:57:18.968563Z
[junit4] 2> 582064 INFO (zkConnectionManagerCallback-1164-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582064 INFO (jetty-launcher-1157-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 582064 INFO (jetty-launcher-1157-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 582064 INFO (zkConnectionManagerCallback-1166-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582070 INFO (jetty-launcher-1157-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 582073 INFO (jetty-launcher-1157-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58203/solr
[junit4] 2> 582076 INFO (jetty-launcher-1157-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58203/solr
[junit4] 2> 582076 INFO (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582078 INFO (jetty-launcher-1157-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58203/solr
[junit4] 2> 582078 INFO (zkConnectionManagerCallback-1174-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582081 INFO (zkConnectionManagerCallback-1180-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582082 INFO (zkConnectionManagerCallback-1178-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582084 INFO (zkConnectionManagerCallback-1182-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582086 INFO (zkConnectionManagerCallback-1184-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582224 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 582224 INFO (jetty-launcher-1157-thread-3) [n:127.0.0.1:58205_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 582225 INFO (jetty-launcher-1157-thread-1) [n:127.0.0.1:58206_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 582226 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:58207_solr
[junit4] 2> 582227 INFO (jetty-launcher-1157-thread-3) [n:127.0.0.1:58205_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58205_solr
[junit4] 2> 582227 INFO (jetty-launcher-1157-thread-1) [n:127.0.0.1:58206_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58206_solr
[junit4] 2> 582228 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.c.Overseer Overseer (id=99351595504959496-127.0.0.1:58207_solr-n_0000000000) starting
[junit4] 2> 582229 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 582229 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 582231 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 582254 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58207_solr
[junit4] 2> 582261 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 582264 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 582267 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 582273 INFO (jetty-launcher-1157-thread-3) [n:127.0.0.1:58205_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58205.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582275 INFO (jetty-launcher-1157-thread-1) [n:127.0.0.1:58206_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58206.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582282 INFO (jetty-launcher-1157-thread-3) [n:127.0.0.1:58205_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58205.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582282 INFO (jetty-launcher-1157-thread-3) [n:127.0.0.1:58205_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58205.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582283 INFO (jetty-launcher-1157-thread-1) [n:127.0.0.1:58206_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58206.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582283 INFO (jetty-launcher-1157-thread-1) [n:127.0.0.1:58206_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58206.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582283 INFO (jetty-launcher-1157-thread-3) [n:127.0.0.1:58205_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node3/.
[junit4] 2> 582284 INFO (jetty-launcher-1157-thread-1) [n:127.0.0.1:58206_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node1/.
[junit4] 2> 582293 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58207.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582301 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58207.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582301 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58207.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 582302 INFO (jetty-launcher-1157-thread-2) [n:127.0.0.1:58207_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node2/.
[junit4] 2> 582348 INFO (zkConnectionManagerCallback-1192-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582351 INFO (zkConnectionManagerCallback-1196-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 582353 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 582354 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:58203/solr ready
[junit4] 2> 582362 INFO (qtp1541579407-4816) [n:127.0.0.1:58205_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&replicationFactor=2&name=.system&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 582370 INFO (OverseerThreadFactory-2366-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.CreateCollectionCmd Create collection .system
[junit4] 2> 582492 INFO (OverseerStateUpdate-99351595504959496-127.0.0.1:58207_solr-n_0000000000) [n:127.0.0.1:58207_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58206/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 582496 INFO (OverseerStateUpdate-99351595504959496-127.0.0.1:58207_solr-n_0000000000) [n:127.0.0.1:58207_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58205/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 582500 INFO (OverseerStateUpdate-99351595504959496-127.0.0.1:58207_solr-n_0000000000) [n:127.0.0.1:58207_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_p4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:58207/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 582726 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n2&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 582726 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 582726 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 582726 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=.system&newCollection=true&name=.system_shard1_replica_p4&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 582727 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 582727 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 582850 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 582850 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 582850 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 582851 INFO (zkCallback-1181-thread-2-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 582851 INFO (zkCallback-1179-thread-2-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 582852 INFO (zkCallback-1183-thread-2-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 583755 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 583766 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 583768 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 583768 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 583773 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 583775 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 583775 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
[junit4] 2> 583775 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema [.system_shard1_replica_p4] Schema name=_system collection or core
[junit4] 2> 583778 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
[junit4] 2> 583779 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 583779 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 583780 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/.system/managed-schema
[junit4] 2> 583780 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ManagedIndexSchema Managed schema znode at /configs/.system/managed-schema already exists - no need to create it
[junit4] 2> 583782 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 583783 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchema Managed schema znode at /configs/.system/managed-schema already exists - no need to create it
[junit4] 2> 583784 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/.system/schema.xml to /configs/.system/schema.xml.bak
[junit4] 2> 583784 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_p4' using configuration from collection .system, trusted=true
[junit4] 2> 583785 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58207.solr.core..system.shard1.replica_p4' (registry 'solr.core..system.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 583785 WARN (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory Error persisting managed schema resource managed-schema
[junit4] 2> org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /configs/.system/schema.xml
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$delete$1(SolrZkClient.java:243)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:242)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.zkUgradeToManagedSchema(ManagedIndexSchemaFactory.java:349)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.upgradeToManagedSchema(ManagedIndexSchemaFactory.java:268)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:186)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:45)
[junit4] 2> at org.apache.solr.schema.IndexSchemaFactory.buildIndexSchema(IndexSchemaFactory.java:75)
[junit4] 2> at org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:119)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:92)
[junit4] 2> at org.apache.solr.core.CoreContainer.getConfigSet(CoreContainer.java:1083)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1035)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] 2> 583785 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n1' using configuration from collection .system, trusted=true
[junit4] 2> 583785 WARN (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchemaFactory Error persisting managed schema resource managed-schema
[junit4] 2> org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /configs/.system/schema.xml
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$delete$1(SolrZkClient.java:243)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:242)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.zkUgradeToManagedSchema(ManagedIndexSchemaFactory.java:349)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.upgradeToManagedSchema(ManagedIndexSchemaFactory.java:268)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:186)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:45)
[junit4] 2> at org.apache.solr.schema.IndexSchemaFactory.buildIndexSchema(IndexSchemaFactory.java:75)
[junit4] 2> at org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:119)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:92)
[junit4] 2> at org.apache.solr.core.CoreContainer.getConfigSet(CoreContainer.java:1083)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1035)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] 2> 583786 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58206.solr.core..system.shard1.replica_n1' (registry 'solr.core..system.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 583786 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n2' using configuration from collection .system, trusted=true
[junit4] 2> 583786 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 583786 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [[.system_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node1/.system_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node1/./.system_shard1_replica_n1/data/]
[junit4] 2> 583786 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58205.solr.core..system.shard1.replica_n2' (registry 'solr.core..system.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 583786 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 583786 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [[.system_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node3/.system_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node3/./.system_shard1_replica_n2/data/]
[junit4] 2> 583785 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 583787 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore [[.system_shard1_replica_p4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node2/.system_shard1_replica_p4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node2/./.system_shard1_replica_p4/data/]
[junit4] 2> 583894 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 583894 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 583895 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
[junit4] 2> 583895 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 583896 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
[junit4] 2> 583896 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 583904 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 583904 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 583906 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
[junit4] 2> 583906 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 583939 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@5ba7ec72[.system_shard1_replica_p4] main]
[junit4] 2> 583940 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@416db18d[.system_shard1_replica_n2] main]
[junit4] 2> 583940 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 583941 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 583941 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 583941 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 583941 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 0 from ZooKeeper
[junit4] 2> 583942 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 583942 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 583942 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 0 from ZooKeeper
[junit4] 2> 583943 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema [.system_shard1_replica_p4] Schema name=_system collection or core
[junit4] 2> 583943 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
[junit4] 2> 583945 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 583946 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 4 ms
[junit4] 2> 583946 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 583946 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 3 ms
[junit4] 2> 583946 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 583946 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 583947 INFO (searcherExecutor-2373-thread-1-processing-n:127.0.0.1:58207_solr x:.system_shard1_replica_p4 s:shard1 c:.system r:core_node6) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore [.system_shard1_replica_p4] Registered new searcher Searcher@5ba7ec72[.system_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 583947 INFO (searcherExecutor-2375-thread-1-processing-n:127.0.0.1:58205_solr x:.system_shard1_replica_n2 s:shard1 c:.system r:core_node5) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [.system_shard1_replica_n2] Registered new searcher Searcher@416db18d[.system_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 583947 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.ZkController .system_shard1_replica_p4 starting background replication from leader
[junit4] 2> 583947 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 583947 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589625530196426752
[junit4] 2> 583948 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 583948 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 583956 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5c07ab27[.system_shard1_replica_n1] main]
[junit4] 2> 583957 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 583957 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 583958 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 583958 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 583958 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 583959 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 583959 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589625530209009664
[junit4] 2> 583960 INFO (searcherExecutor-2374-thread-1-processing-n:127.0.0.1:58206_solr x:.system_shard1_replica_n1 s:shard1 c:.system r:core_node3) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [.system_shard1_replica_n1] Registered new searcher Searcher@5c07ab27[.system_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 584067 INFO (zkCallback-1181-thread-2-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584067 INFO (zkCallback-1183-thread-2-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584067 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584067 INFO (zkCallback-1179-thread-2-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584067 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584067 INFO (zkCallback-1181-thread-3-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584469 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 584469 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 584469 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:58205/solr/.system_shard1_replica_n2/
[junit4] 2> 584469 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=.system_shard1_replica_n2 url=http://127.0.0.1:58205/solr START replicas=[http://127.0.0.1:58206/solr/.system_shard1_replica_n1/] nUpdates=100
[junit4] 2> 584472 INFO (qtp1417806960-4826) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 584776 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=.system_shard1_replica_n2 url=http://127.0.0.1:58205/solr DONE. We have no versions. sync failed.
[junit4] 2> 584776 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 584776 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 584776 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 584785 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:58205/solr/.system_shard1_replica_n2/ shard1
[junit4] 2> 584896 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584896 INFO (zkCallback-1181-thread-3-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584896 INFO (zkCallback-1181-thread-2-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584896 INFO (zkCallback-1179-thread-2-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584896 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584896 INFO (zkCallback-1183-thread-2-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 584947 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 584951 INFO (qtp1541579407-4819) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n2&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2226
[junit4] 2> 584954 INFO (qtp1195980972-4828) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=.system&newCollection=true&name=.system_shard1_replica_p4&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2227
[junit4] 2> 584969 INFO (qtp1417806960-4824) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2242
[junit4] 2> 584975 INFO (qtp1541579407-4816) [n:127.0.0.1:58205_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 585001 INFO (indexFetcher-2388-thread-1) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.IndexFetcher Replica core_node5 is leader but it's state is down, skipping replication
[junit4] 2> 585076 INFO (zkCallback-1181-thread-2-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 585076 INFO (zkCallback-1183-thread-2-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 585076 INFO (zkCallback-1179-thread-2-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 585076 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 585076 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 585076 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 585983 INFO (qtp1541579407-4816) [n:127.0.0.1:58205_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=1&replicationFactor=2&name=.system&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=3621
[junit4] 2> 585990 INFO (qtp1541579407-4821) [n:127.0.0.1:58205_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 585994 INFO (qtp1417806960-4835) [n:127.0.0.1:58206_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 585999 INFO (qtp1195980972-4830) [n:127.0.0.1:58207_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 586061 INFO (TEST-TestClusterStateProvider.testAutoScalingConfig-seed#[819A134696D623FA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAutoScalingConfig
[junit4] 2> 586065 INFO (TEST-TestClusterStateProvider.testAutoScalingConfig-seed#[819A134696D623FA]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAutoScalingConfig
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestClusterStateProvider -Dtests.method=testAutoScalingConfig -Dtests.seed=819A134696D623FA -Dtests.slow=true -Dtests.locale=mg-MG -Dtests.timezone=CET -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.03s J0 | TestClusterStateProvider.testAutoScalingConfig <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
[junit4] > "cluster-preferences":[{"maximize":"freedisk"}],
[junit4] > "triggers":{".auto_add_replicas":{
[junit4] > "name":".auto_add_replicas",
[junit4] > "event":"nodeLost",
[junit4] > "waitFor":30,
[junit4] > "actions":[
[junit4] > {
[junit4] > "name":"auto_add_replicas_plan",
[junit4] > "class":"solr.AutoAddReplicasPlanAction"},
[junit4] > {
[junit4] > "name":"execute_plan",
[junit4] > "class":"solr.ExecutePlanAction"}],
[junit4] > "enabled":true}},
[junit4] > "listeners":{".auto_add_replicas.system":{
[junit4] > "trigger":".auto_add_replicas",
[junit4] > "afterAction":[],
[junit4] > "stage":[
[junit4] > "STARTED",
[junit4] > "ABORTED",
[junit4] > "SUCCEEDED",
[junit4] > "FAILED",
[junit4] > "BEFORE_ACTION",
[junit4] > "AFTER_ACTION",
[junit4] > "IGNORED"],
[junit4] > "class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
[junit4] > "beforeAction":[]}},
[junit4] > "properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
[junit4] > "cluster-preferences":[{"maximize":"freedisk"}],
[junit4] > "triggers":{".auto_add_replicas":{
[junit4] > "name":".auto_add_replicas",
[junit4] > "event":"nodeLost",
[junit4] > "waitFor":30,
[junit4] > "actions":[
[junit4] > {
[junit4] > "name":"auto_add_replicas_plan",
[junit4] > "class":"solr.AutoAddReplicasPlanAction"},
[junit4] > {
[junit4] > "name":"execute_plan",
[junit4] > "class":"solr.ExecutePlanAction"}],
[junit4] > "enabled":true}},
[junit4] > "listeners":{".auto_add_replicas.system":{
[junit4] > "trigger":".auto_add_replicas",
[junit4] > "afterAction":[],
[junit4] > "stage":[
[junit4] > "STARTED",
[junit4] > "ABORTED",
[junit4] > "SUCCEEDED",
[junit4] > "FAILED",
[junit4] > "BEFORE_ACTION",
[junit4] > "AFTER_ACTION",
[junit4] > "IGNORED"],
[junit4] > "class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
[junit4] > "beforeAction":[]}},
[junit4] > "properties":{}}>
[junit4] > at __randomizedtesting.SeedInfo.seed([819A134696D623FA:BE1210EE8180D31D]:0)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:564)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:844)
[junit4] 2> 586097 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAddRemoveNode
[junit4] 2> 586098 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 586100 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2eb8be4d{/solr,null,AVAILABLE}
[junit4] 2> 586101 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@58951abd{HTTP/1.1,[http/1.1]}{127.0.0.1:58243}
[junit4] 2> 586101 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.e.j.s.Server Started @587841ms
[junit4] 2> 586101 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58243}
[junit4] 2> 586101 ERROR (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 586101 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 586101 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 586101 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 586101 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-15T02:57:23.006692Z
[junit4] 2> 586104 INFO (zkConnectionManagerCallback-1198-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 586104 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 586124 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58203/solr
[junit4] 2> 586125 INFO (zkConnectionManagerCallback-1202-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 586128 INFO (zkConnectionManagerCallback-1204-thread-1-processing-n:127.0.0.1:58243_solr) [n:127.0.0.1:58243_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 586138 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [n:127.0.0.1:58243_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 586142 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [n:127.0.0.1:58243_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 586145 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [n:127.0.0.1:58243_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58243_solr
[junit4] 2> 586147 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 586147 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 586147 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 586147 INFO (zkCallback-1203-thread-1-processing-n:127.0.0.1:58243_solr) [n:127.0.0.1:58243_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 586147 INFO (zkCallback-1195-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 586180 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [n:127.0.0.1:58243_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58243.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 586189 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [n:127.0.0.1:58243_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58243.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 586189 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [n:127.0.0.1:58243_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58243.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@74102bd5
[junit4] 2> 586190 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [n:127.0.0.1:58243_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001/tempDir-001/node4/.
[junit4] 2> 586392 INFO (OverseerCollectionConfigSetProcessor-99351595504959496-127.0.0.1:58207_solr-n_0000000000) [n:127.0.0.1:58207_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 586442 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@29c5aa16{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 586443 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1276626785
[junit4] 2> 586443 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 586444 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3bb58d51: rootName = solr_58206, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@7bf3f616
[junit4] 2> 586465 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 586465 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4005c8f3: rootName = solr_58206, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@235d5498
[junit4] 2> 586476 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 586476 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5624b2ef: rootName = solr_58206, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@7b3d9a2e
[junit4] 2> 586478 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:58206_solr
[junit4] 2> 586480 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 586480 INFO (zkCallback-1203-thread-1-processing-n:127.0.0.1:58243_solr) [n:127.0.0.1:58243_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 586481 INFO (zkCallback-1183-thread-1-processing-n:127.0.0.1:58206_solr) [n:127.0.0.1:58206_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 586481 INFO (zkCallback-1195-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 586481 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 586481 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 586481 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [.system_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@68b9fd4e
[junit4] 2> 586481 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_n1, tag=1757019470
[junit4] 2> 586481 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@8dee18e: rootName = solr_58206, domain = solr.core..system.shard1.replica_n1, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@15478197
[junit4] 2> 586488 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:58206_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=1757019470
[junit4] 2> 586491 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@32cf1c5a{/solr,null,UNAVAILABLE}
[junit4] 2> 586591 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 586591 INFO (zkCallback-1179-thread-2-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 586591 INFO (zkCallback-1181-thread-2-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 586591 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 586704 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[819A134696D623FA]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAddRemoveNode
[junit4] 2> 586704 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 586707 INFO (jetty-closer-1158-thread-3) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@58951abd{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 586707 INFO (jetty-closer-1158-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@6ebda1a5{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-3) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1059702921
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1438817295
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-3) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@9530652: rootName = solr_58243, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6dbb01c0
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@39f3eb09: rootName = solr_58205, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@632af336
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@66a49ca4{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=138782137
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 586708 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@73c09b26: rootName = solr_58207, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@4e3e6c1f
[junit4] 2> 586742 INFO (jetty-closer-1158-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 586742 INFO (jetty-closer-1158-thread-3) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@17084042: rootName = solr_58243, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@235d5498
[junit4] 2> 586743 INFO (jetty-closer-1158-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 586743 INFO (jetty-closer-1158-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7bd0e6b: rootName = solr_58205, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@235d5498
[junit4] 2> 586756 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 586756 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5f2d2b57: rootName = solr_58207,
[...truncated too long message...]
kController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:58205_solr
[junit4] 2> 586786 INFO (jetty-closer-1158-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 586787 INFO (jetty-closer-1158-thread-3) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 586788 INFO (zkCallback-1203-thread-1-processing-n:127.0.0.1:58243_solr) [n:127.0.0.1:58243_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 586788 INFO (zkCallback-1181-thread-2-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 586788 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (2)
[junit4] 2> 586788 INFO (jetty-closer-1158-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 586789 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:58207_solr
[junit4] 2> 586789 INFO (coreCloseExecutor-2402-thread-1) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [.system_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@422dd697
[junit4] 2> 586789 INFO (coreCloseExecutor-2402-thread-1) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_n2, tag=1110300311
[junit4] 2> 586789 INFO (coreCloseExecutor-2402-thread-1) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@561d1c7b: rootName = solr_58205, domain = solr.core..system.shard1.replica_n2, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@6d9dc0ce
[junit4] 2> 586790 INFO (zkCallback-1181-thread-1-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 586790 INFO (zkCallback-1179-thread-2-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 586792 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 586793 INFO (coreCloseExecutor-2403-thread-1) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore [.system_shard1_replica_p4] CLOSING SolrCore org.apache.solr.core.SolrCore@1929d153
[junit4] 2> 586793 INFO (coreCloseExecutor-2403-thread-1) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_p4, tag=422170963
[junit4] 2> 586793 INFO (coreCloseExecutor-2403-thread-1) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3eb7050c: rootName = solr_58207, domain = solr.core..system.shard1.replica_p4, service url = null, agent id = null] for registry solr.core..system.shard1.replica_p4 / com.codahale.metrics.MetricRegistry@5981d714
[junit4] 2> 586793 INFO (zkCallback-1181-thread-2-processing-n:127.0.0.1:58207_solr) [n:127.0.0.1:58207_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 586793 INFO (zkCallback-1179-thread-1-processing-n:127.0.0.1:58205_solr) [n:127.0.0.1:58205_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 586807 INFO (coreCloseExecutor-2402-thread-1) [n:127.0.0.1:58205_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=1110300311
[junit4] 2> 586809 INFO (jetty-closer-1158-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@23d5e0e7{/solr,null,UNAVAILABLE}
[junit4] 2> 586810 INFO (coreCloseExecutor-2403-thread-1) [n:127.0.0.1:58207_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=422170963
[junit4] 2> 586811 INFO (jetty-closer-1158-thread-2) [ ] o.a.s.c.Overseer Overseer (id=99351595504959496-127.0.0.1:58207_solr-n_0000000000) closing
[junit4] 2> 586812 INFO (OverseerStateUpdate-99351595504959496-127.0.0.1:58207_solr-n_0000000000) [n:127.0.0.1:58207_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:58207_solr
[junit4] 2> 586812 WARN (OverseerAutoScalingTriggerThread-99351595504959496-127.0.0.1:58207_solr-n_0000000000) [n:127.0.0.1:58207_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 586814 INFO (jetty-closer-1158-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@53816dc3{/solr,null,UNAVAILABLE}
[junit4] 2> 588298 WARN (zkCallback-1203-thread-1-processing-n:127.0.0.1:58243_solr) [n:127.0.0.1:58243_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 588299 INFO (jetty-closer-1158-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2eb8be4d{/solr,null,UNAVAILABLE}
[junit4] 2> 588300 ERROR (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 588300 INFO (SUITE-TestClusterStateProvider-seed#[819A134696D623FA]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:58203 58203
[junit4] 2> 588318 INFO (Thread-889) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:58203 58203
[junit4] 2> 588520 WARN (Thread-889) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 3 /solr/configs/.system
[junit4] 2> 3 /solr/configs/.system/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 20 /solr/collections/.system/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/99351595504959494-127.0.0.1:58205_solr-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_819A134696D623FA-001
[junit4] 2> Jan 15, 2018 2:57:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=mg-MG, timezone=CET
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 9 (64-bit)/cpus=3,threads=1,free=79704096,total=270532608
[junit4] 2> NOTE: All tests run in this JVM: [CdcrReplicationDistributedZkTest, TestInPlaceUpdatesStandalone, HighlighterTest, PeerSyncReplicationTest, TestGraphTermsQParserPlugin, ConvertedLegacyTest, HLLSerializationTest, DistributedQueryElevationComponentTest, DistanceUnitsTest, TestIndexingPerformance, DocExpirationUpdateProcessorFactoryTest, SignatureUpdateProcessorFactoryTest, VMParamsZkACLAndCredentialsProvidersTest, TestInitParams, TestQuerySenderListener, TestCollapseQParserPlugin, TestImpersonationWithHadoopAuth, CleanupOldIndexTest, TestPKIAuthenticationPlugin, ScriptEngineTest, ShufflingReplicaListTransformerTest, FastVectorHighlighterTest, SolrXmlInZkTest, TestTlogReplica, TestRecovery, SolrCmdDistributorTest, TestClassicSimilarityFactory, ChaosMonkeySafeLeaderWithPullReplicasTest, TestSchemaResource, SpatialFilterTest, TestSlowCompositeReaderWrapper, PrimUtilsTest, TestRestManager, TestReversedWildcardFilterFactory, TestLegacyFieldCache, OverseerTest, VersionInfoTest, TestNonDefinedSimilarityFactory, TestStressInPlaceUpdates, HdfsBasicDistributedZk2Test, DistributedQueryComponentCustomSortTest, TestCustomStream, TestDynamicFieldResource, NotRequiredUniqueKeyTest, TestRandomRequestDistribution, ReplicationFactorTest, ChangedSchemaMergeTest, CoreAdminCreateDiscoverTest, HdfsChaosMonkeyNothingIsSafeTest, TestUtils, OverseerModifyCollectionTest, TestEmbeddedSolrServerSchemaAPI, TestWriterPerf, HdfsThreadLeakTest, SparseHLLTest, TestSystemCollAutoCreate, TestImplicitCoreProperties, AsyncCallRequestStatusResponseTest, TestJavabinTupleStreamParser, BadCopyFieldTest, TestJsonFacets, SortSpecParsingTest, DistributedFacetPivotSmallTest, TestPointFields, SolrInfoBeanTest, SharedFSAutoReplicaFailoverTest, CdcrUpdateLogTest, ResponseLogComponentTest, TestClusterStateProvider]
[junit4] Completed [134/767 (1!)] on J0 in 7.40s, 2 tests, 1 failure <<< FAILURES!
[...truncated 42859 lines...]
[JENKINS] Lucene-Solr-7.x-MacOSX (64bit/jdk1.8.0) - Build # 399 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-MacOSX/399/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig
Error Message:
expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}>
Stack Trace:
java.lang.AssertionError: expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}>
at __randomizedtesting.SeedInfo.seed([C946F8ADA3128640:F6CEFB05B44476A7]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12259 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider
[junit4] 2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/init-core-data-001
[junit4] 2> 994228 WARN (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=12 numCloses=12
[junit4] 2> 994229 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 994232 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN) w/ MAC_OS_X supressed clientAuth
[junit4] 2> 994233 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 994233 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.c.a.s.TestClusterStateProvider ####### Using simulated components? true
[junit4] 2> 994233 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001
[junit4] 2> 994233 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 994234 INFO (Thread-1658) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 994234 INFO (Thread-1658) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 994236 ERROR (Thread-1658) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 994334 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:51930
[junit4] 2> 994352 INFO (zkConnectionManagerCallback-1629-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994357 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e688540000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994359 INFO (jetty-launcher-1626-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 994359 INFO (jetty-launcher-1626-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 994360 INFO (jetty-launcher-1626-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 994361 INFO (jetty-launcher-1626-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@16235325{/solr,null,AVAILABLE}
[junit4] 2> 994369 INFO (jetty-launcher-1626-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66b7fdf1{/solr,null,AVAILABLE}
[junit4] 2> 994369 INFO (jetty-launcher-1626-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@67491d60{HTTP/1.1,[http/1.1]}{127.0.0.1:51933}
[junit4] 2> 994369 INFO (jetty-launcher-1626-thread-3) [ ] o.e.j.s.Server Started @1001113ms
[junit4] 2> 994369 INFO (jetty-launcher-1626-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51933}
[junit4] 2> 994369 ERROR (jetty-launcher-1626-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 994369 INFO (jetty-launcher-1626-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4cb45168{HTTP/1.1,[http/1.1]}{127.0.0.1:51932}
[junit4] 2> 994370 INFO (jetty-launcher-1626-thread-2) [ ] o.e.j.s.Server Started @1001114ms
[junit4] 2> 994370 INFO (jetty-launcher-1626-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51932}
[junit4] 2> 994370 INFO (jetty-launcher-1626-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 994370 INFO (jetty-launcher-1626-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 994370 INFO (jetty-launcher-1626-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 994370 INFO (jetty-launcher-1626-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T13:40:37.210Z
[junit4] 2> 994370 ERROR (jetty-launcher-1626-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 994370 INFO (jetty-launcher-1626-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@48883423{/solr,null,AVAILABLE}
[junit4] 2> 994371 INFO (jetty-launcher-1626-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 994371 INFO (jetty-launcher-1626-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 994371 INFO (jetty-launcher-1626-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 994371 INFO (jetty-launcher-1626-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T13:40:37.211Z
[junit4] 2> 994373 INFO (jetty-launcher-1626-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3b338bd0{HTTP/1.1,[http/1.1]}{127.0.0.1:51934}
[junit4] 2> 994373 INFO (jetty-launcher-1626-thread-1) [ ] o.e.j.s.Server Started @1001116ms
[junit4] 2> 994373 INFO (jetty-launcher-1626-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51934}
[junit4] 2> 994373 ERROR (jetty-launcher-1626-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 994376 INFO (jetty-launcher-1626-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 994376 INFO (jetty-launcher-1626-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 994377 INFO (jetty-launcher-1626-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 994377 INFO (jetty-launcher-1626-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T13:40:37.217Z
[junit4] 2> 994377 INFO (zkConnectionManagerCallback-1631-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994380 INFO (jetty-launcher-1626-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 994380 INFO (zkConnectionManagerCallback-1633-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994381 INFO (zkConnectionManagerCallback-1635-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994381 INFO (jetty-launcher-1626-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 994381 INFO (jetty-launcher-1626-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 994387 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e688540003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994391 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e688540002, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994391 INFO (jetty-launcher-1626-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51930/solr
[junit4] 2> 994392 INFO (jetty-launcher-1626-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51930/solr
[junit4] 2> 994393 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e688540001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994395 INFO (zkConnectionManagerCallback-1639-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994395 INFO (jetty-launcher-1626-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51930/solr
[junit4] 2> 994396 INFO (zkConnectionManagerCallback-1643-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994397 INFO (zkConnectionManagerCallback-1647-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994398 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e688540004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994399 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e688540005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994400 INFO (zkConnectionManagerCallback-1649-thread-1-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994401 INFO (zkConnectionManagerCallback-1651-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994401 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e688540006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994404 INFO (zkConnectionManagerCallback-1653-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994576 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 994576 INFO (jetty-launcher-1626-thread-2) [n:127.0.0.1:51932_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 994576 INFO (jetty-launcher-1626-thread-3) [n:127.0.0.1:51933_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 994576 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51934_solr
[junit4] 2> 994578 INFO (jetty-launcher-1626-thread-2) [n:127.0.0.1:51932_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51932_solr
[junit4] 2> 994578 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.c.Overseer Overseer (id=99348462769012743-127.0.0.1:51934_solr-n_0000000000) starting
[junit4] 2> 994578 INFO (jetty-launcher-1626-thread-3) [n:127.0.0.1:51933_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51933_solr
[junit4] 2> 994580 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 994581 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 994581 INFO (zkCallback-1648-thread-1-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 994582 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 994594 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 994604 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51934_solr
[junit4] 2> 994605 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 994605 INFO (zkCallback-1648-thread-1-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 994616 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 994684 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51934.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994691 INFO (jetty-launcher-1626-thread-3) [n:127.0.0.1:51933_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51933.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994715 INFO (jetty-launcher-1626-thread-3) [n:127.0.0.1:51933_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51933.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994717 INFO (jetty-launcher-1626-thread-3) [n:127.0.0.1:51933_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51933.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994720 INFO (jetty-launcher-1626-thread-3) [n:127.0.0.1:51933_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node3/.
[junit4] 2> 994726 INFO (jetty-launcher-1626-thread-2) [n:127.0.0.1:51932_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51932.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994734 INFO (jetty-launcher-1626-thread-2) [n:127.0.0.1:51932_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51932.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994734 INFO (jetty-launcher-1626-thread-2) [n:127.0.0.1:51932_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51932.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994736 INFO (jetty-launcher-1626-thread-2) [n:127.0.0.1:51932_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node2/.
[junit4] 2> 994738 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51934.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994738 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51934.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 994742 INFO (jetty-launcher-1626-thread-1) [n:127.0.0.1:51934_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node1/.
[junit4] 2> 994842 INFO (zkConnectionManagerCallback-1661-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994846 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f4e68854000a, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 994852 INFO (zkConnectionManagerCallback-1665-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 994853 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 994855 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:51930/solr ready
[junit4] 2> 994862 INFO (qtp410541803-8421) [n:127.0.0.1:51933_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&replicationFactor=2&name=.system&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 994878 INFO (OverseerThreadFactory-3873-thread-1-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.CreateCollectionCmd Create collection .system
[junit4] 2> 994994 INFO (OverseerStateUpdate-99348462769012743-127.0.0.1:51934_solr-n_0000000000) [n:127.0.0.1:51934_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:51932/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 995001 INFO (OverseerStateUpdate-99348462769012743-127.0.0.1:51934_solr-n_0000000000) [n:127.0.0.1:51934_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:51934/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 995006 INFO (OverseerStateUpdate-99348462769012743-127.0.0.1:51934_solr-n_0000000000) [n:127.0.0.1:51934_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_p4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:51933/solr",
[junit4] 2> "type":"PULL",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 995221 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=.system&newCollection=true&name=.system_shard1_replica_p4&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=PULL
[junit4] 2> 995221 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 995221 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n2&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 995222 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 995222 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 995222 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 995339 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 995339 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 995340 INFO (zkCallback-1648-thread-1-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 995340 INFO (zkCallback-1652-thread-2-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 995341 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 995341 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 996249 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 996249 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 996258 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 996259 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 996259 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 996261 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema [.system_shard1_replica_p4] Schema name=_system collection or core
[junit4] 2> 996261 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema [.system_shard1_replica_n2] Schema name=_system collection or core
[junit4] 2> 996262 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory The schema is configured as managed, but managed schema resource managed-schema not found - loading non-managed schema schema.xml instead
[junit4] 2> 996264 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
[junit4] 2> 996266 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 996266 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 996272 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchema Created and persisted managed schema znode at /configs/.system/managed-schema
[junit4] 2> 996272 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ManagedIndexSchema Managed schema znode at /configs/.system/managed-schema already exists - no need to create it
[junit4] 2> 996272 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 996273 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchema Managed schema znode at /configs/.system/managed-schema already exists - no need to create it
[junit4] 2> 996276 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ManagedIndexSchemaFactory After upgrading to managed schema in ZooKeeper, renamed the non-managed schema /configs/.system/schema.xml to /configs/.system/schema.xml.bak
[junit4] 2> 996276 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n2' using configuration from collection .system, trusted=true
[junit4] 2> 996277 WARN (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ManagedIndexSchemaFactory Error persisting managed schema resource managed-schema
[junit4] 2> org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /configs/.system/schema.xml
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$delete$1(SolrZkClient.java:243)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:242)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.zkUgradeToManagedSchema(ManagedIndexSchemaFactory.java:349)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.upgradeToManagedSchema(ManagedIndexSchemaFactory.java:268)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:186)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:45)
[junit4] 2> at org.apache.solr.schema.IndexSchemaFactory.buildIndexSchema(IndexSchemaFactory.java:75)
[junit4] 2> at org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:119)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:92)
[junit4] 2> at org.apache.solr.core.CoreContainer.getConfigSet(CoreContainer.java:1083)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1035)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 996277 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_p4' using configuration from collection .system, trusted=true
[junit4] 2> 996277 WARN (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ManagedIndexSchemaFactory Error persisting managed schema resource managed-schema
[junit4] 2> org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /configs/.system/schema.xml
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$delete$1(SolrZkClient.java:243)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:242)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.zkUgradeToManagedSchema(ManagedIndexSchemaFactory.java:349)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.upgradeToManagedSchema(ManagedIndexSchemaFactory.java:268)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:186)
[junit4] 2> at org.apache.solr.schema.ManagedIndexSchemaFactory.create(ManagedIndexSchemaFactory.java:45)
[junit4] 2> at org.apache.solr.schema.IndexSchemaFactory.buildIndexSchema(IndexSchemaFactory.java:75)
[junit4] 2> at org.apache.solr.core.ConfigSetService.createIndexSchema(ConfigSetService.java:119)
[junit4] 2> at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:92)
[junit4] 2> at org.apache.solr.core.CoreContainer.getConfigSet(CoreContainer.java:1083)
[junit4] 2> at org.apache.solr.core.CoreContainer.createFromDescriptor(CoreContainer.java:1035)
[junit4] 2> at org.apache.solr.core.CoreContainer.create(CoreContainer.java:950)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:91)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:384)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:389)
[junit4] 2> at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:174)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 996278 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore '.system_shard1_replica_n1' using configuration from collection .system, trusted=true
[junit4] 2> 996277 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51934.solr.core..system.shard1.replica_n2' (registry 'solr.core..system.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 996278 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51933.solr.core..system.shard1.replica_p4' (registry 'solr.core..system.shard1.replica_p4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 996278 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 996278 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 996278 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51932.solr.core..system.shard1.replica_n1' (registry 'solr.core..system.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 996278 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore [[.system_shard1_replica_p4] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node3/.system_shard1_replica_p4], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node3/./.system_shard1_replica_p4/data/]
[junit4] 2> 996278 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 996278 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [[.system_shard1_replica_n2] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node1/.system_shard1_replica_n2], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node1/./.system_shard1_replica_n2/data/]
[junit4] 2> 996278 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [[.system_shard1_replica_n1] ] Opening new SolrCore at [/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node2/.system_shard1_replica_n1], dataDir=[/Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node2/./.system_shard1_replica_n1/data/]
[junit4] 2> 996347 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
[junit4] 2> 996347 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 996347 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 996347 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 996348 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 996348 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 996348 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
[junit4] 2> 996348 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 996349 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if 1 uncommited docs
[junit4] 2> 996349 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 996350 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3faf4d14[.system_shard1_replica_n2] main]
[junit4] 2> 996350 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.SolrIndexSearcher Opening [Searcher@d371106[.system_shard1_replica_p4] main]
[junit4] 2> 996350 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@548b8a37[.system_shard1_replica_n1] main]
[junit4] 2> 996351 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 996351 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 996351 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/.system
[junit4] 2> 996351 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 996352 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 996352 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 996352 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 996352 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/.system
[junit4] 2> 996352 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/.system/managed-schema
[junit4] 2> 996352 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 996352 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 0 from ZooKeeper
[junit4] 2> 996353 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Retrieved schema version 0 from ZooKeeper
[junit4] 2> 996353 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 996353 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589575406524039168
[junit4] 2> 996354 INFO (searcherExecutor-3880-thread-1-processing-n:127.0.0.1:51934_solr x:.system_shard1_replica_n2 s:shard1 c:.system r:core_node5) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [.system_shard1_replica_n2] Registered new searcher Searcher@3faf4d14[.system_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 996357 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema [.system_shard1_replica_p4] Schema name=_system collection or core
[junit4] 2> 996357 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema [.system_shard1_replica_n1] Schema name=_system collection or core
[junit4] 2> 996362 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 996362 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 9 ms
[junit4] 2> 996362 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema _system collection or core/1.1 with uniqueid field id
[junit4] 2> 996362 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Finished refreshing schema in 9 ms
[junit4] 2> 996362 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 996362 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 996363 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589575406534524928
[junit4] 2> 996363 INFO (searcherExecutor-3882-thread-1-processing-n:127.0.0.1:51932_solr x:.system_shard1_replica_n1 s:shard1 c:.system r:core_node3) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.SolrCore [.system_shard1_replica_n1] Registered new searcher Searcher@548b8a37[.system_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 996363 INFO (searcherExecutor-3881-thread-1-processing-n:127.0.0.1:51933_solr x:.system_shard1_replica_p4 s:shard1 c:.system r:core_node6) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore [.system_shard1_replica_p4] Registered new searcher Searcher@d371106[.system_shard1_replica_p4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 996364 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9998ms
[junit4] 2> 996364 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.ZkController .system_shard1_replica_p4 starting background replication from leader
[junit4] 2> 996364 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.ReplicateFromLeader Will start replication from leader with poll interval: 00:00:03
[junit4] 2> 996366 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 3000ms
[junit4] 2> 996366 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 996474 INFO (zkCallback-1652-thread-2-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 996474 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 996474 INFO (zkCallback-1648-thread-3-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 996474 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 996474 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 996474 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 996871 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 996871 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 996871 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51934/solr/.system_shard1_replica_n2/
[junit4] 2> 996871 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=.system_shard1_replica_n2 url=http://127.0.0.1:51934/solr START replicas=[http://127.0.0.1:51932/solr/.system_shard1_replica_n1/] nUpdates=100
[junit4] 2> 996872 INFO (qtp1878001526-8407) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.c.S.Request [.system_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 997174 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=.system_shard1_replica_n2 url=http://127.0.0.1:51934/solr DONE. We have no versions. sync failed.
[junit4] 2> 997174 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 997174 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 997174 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 997186 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51934/solr/.system_shard1_replica_n2/ shard1
[junit4] 2> 997290 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997290 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997290 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997290 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997290 INFO (zkCallback-1652-thread-2-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997290 INFO (zkCallback-1648-thread-3-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997308 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 997309 INFO (indexFetcher-3895-thread-1) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.h.IndexFetcher Replica core_node5 is leader but it's state is down, skipping replication
[junit4] 2> 997311 INFO (qtp1877871660-8414) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n2&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2089
[junit4] 2> 997384 INFO (qtp410541803-8423) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=.system&newCollection=true&name=.system_shard1_replica_p4&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=PULL} status=0 QTime=2163
[junit4] 2> 997385 INFO (qtp1878001526-8406) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=.system&newCollection=true&name=.system_shard1_replica_n1&action=CREATE&numShards=1&collection=.system&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2164
[junit4] 2> 997393 INFO (qtp410541803-8421) [n:127.0.0.1:51933_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 997500 INFO (zkCallback-1652-thread-2-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997500 INFO (zkCallback-1648-thread-3-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997500 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997500 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997500 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 997500 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 998396 INFO (qtp410541803-8421) [n:127.0.0.1:51933_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=1&replicationFactor=2&name=.system&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=3533
[junit4] 2> 998403 INFO (qtp410541803-8425) [n:127.0.0.1:51933_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 998408 INFO (qtp1878001526-8411) [n:127.0.0.1:51932_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 998413 INFO (qtp1877871660-8415) [n:127.0.0.1:51934_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=1
[junit4] 2> 998485 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAddRemoveNode
[junit4] 2> 998485 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 998487 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@62f99c7a{/solr,null,AVAILABLE}
[junit4] 2> 998487 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@372f20c5{HTTP/1.1,[http/1.1]}{127.0.0.1:51964}
[junit4] 2> 998487 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.e.j.s.Server Started @1005231ms
[junit4] 2> 998487 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51964}
[junit4] 2> 998488 ERROR (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 998488 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 998488 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 998488 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 998488 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T13:40:41.328Z
[junit4] 2> 998490 INFO (zkConnectionManagerCallback-1667-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 998490 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 998495 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:51930/solr
[junit4] 2> 998497 INFO (zkConnectionManagerCallback-1671-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 998509 INFO (zkConnectionManagerCallback-1673-thread-1-processing-n:127.0.0.1:51964_solr) [n:127.0.0.1:51964_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 998518 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [n:127.0.0.1:51964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 998522 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [n:127.0.0.1:51964_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 998525 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [n:127.0.0.1:51964_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51964_solr
[junit4] 2> 998526 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 998526 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 998527 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 998527 INFO (zkCallback-1672-thread-1-processing-n:127.0.0.1:51964_solr) [n:127.0.0.1:51964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 998527 INFO (zkCallback-1664-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 998622 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [n:127.0.0.1:51964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51964.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 998628 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [n:127.0.0.1:51964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51964.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 998628 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [n:127.0.0.1:51964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_51964.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@79c0adad
[junit4] 2> 998630 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [n:127.0.0.1:51964_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001/tempDir-001/node4/.
[junit4] 2> 998877 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@67491d60{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 998877 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1368856218
[junit4] 2> 998878 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 998878 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3d78e4ae: rootName = solr_51933, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@9b88310
[junit4] 2> 998885 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 998885 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@79f6c6b1: rootName = solr_51933, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@61903954
[junit4] 2> 998890 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 998890 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@50d662f5: rootName = solr_51933, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@132d6c0e
[junit4] 2> 998892 INFO (OverseerCollectionConfigSetProcessor-99348462769012743-127.0.0.1:51934_solr-n_0000000000) [n:127.0.0.1:51934_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 998893 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:51933_solr
[junit4] 2> 998895 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 998895 INFO (zkCallback-1672-thread-1-processing-n:127.0.0.1:51964_solr) [n:127.0.0.1:51964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 998895 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 998895 INFO (zkCallback-1652-thread-1-processing-n:127.0.0.1:51933_solr) [n:127.0.0.1:51933_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 998895 INFO (zkCallback-1664-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 998897 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 998898 INFO (coreCloseExecutor-3906-thread-1) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore [.system_shard1_replica_p4] CLOSING SolrCore org.apache.solr.core.SolrCore@7c3e5c1e
[junit4] 2> 998898 INFO (coreCloseExecutor-3906-thread-1) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_p4, tag=2084461598
[junit4] 2> 998898 INFO (coreCloseExecutor-3906-thread-1) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@16c8ad5b: rootName = solr_51933, domain = solr.core..system.shard1.replica_p4, service url = null, agent id = null] for registry solr.core..system.shard1.replica_p4 / com.codahale.metrics.MetricRegistry@6fdeb963
[junit4] 2> 998907 INFO (coreCloseExecutor-3906-thread-1) [n:127.0.0.1:51933_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=2084461598
[junit4] 2> 998911 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@66b7fdf1{/solr,null,UNAVAILABLE}
[junit4] 2> 998998 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 998998 INFO (zkCallback-1650-thread-1-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 998998 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 998998 INFO (zkCallback-1648-thread-1-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/.system/state.json] for collection [.system] has occurred - updating... (live nodes size: [3])
[junit4] 2> 999116 INFO (TEST-TestClusterStateProvider.testAddRemoveNode-seed#[C946F8ADA3128640]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAddRemoveNode
[junit4] 2> 999144 INFO (TEST-TestClusterStateProvider.testAutoScalingConfig-seed#[C946F8ADA3128640]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAutoScalingConfig
[junit4] 2> 999146 INFO (TEST-TestClusterStateProvider.testAutoScalingConfig-seed#[C946F8ADA3128640]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testAutoScalingConfig
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestClusterStateProvider -Dtests.method=testAutoScalingConfig -Dtests.seed=C946F8ADA3128640 -Dtests.slow=true -Dtests.locale=ar-LB -Dtests.timezone=Etc/GMT+8 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.03s J0 | TestClusterStateProvider.testAutoScalingConfig <<<
[junit4] > Throwable #1: java.lang.AssertionError: expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
[junit4] > "cluster-preferences":[{"maximize":"freedisk"}],
[junit4] > "triggers":{".auto_add_replicas":{
[junit4] > "name":".auto_add_replicas",
[junit4] > "event":"nodeLost",
[junit4] > "waitFor":30,
[junit4] > "actions":[
[junit4] > {
[junit4] > "name":"auto_add_replicas_plan",
[junit4] > "class":"solr.AutoAddReplicasPlanAction"},
[junit4] > {
[junit4] > "name":"execute_plan",
[junit4] > "class":"solr.ExecutePlanAction"}],
[junit4] > "enabled":true}},
[junit4] > "listeners":{".auto_add_replicas.system":{
[junit4] > "trigger":".auto_add_replicas",
[junit4] > "afterAction":[],
[junit4] > "stage":[
[junit4] > "STARTED",
[junit4] > "ABORTED",
[junit4] > "SUCCEEDED",
[junit4] > "FAILED",
[junit4] > "BEFORE_ACTION",
[junit4] > "AFTER_ACTION",
[junit4] > "IGNORED"],
[junit4] > "class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
[junit4] > "beforeAction":[]}},
[junit4] > "properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
[junit4] > "cluster-preferences":[{"maximize":"freedisk"}],
[junit4] > "triggers":{".auto_add_replicas":{
[junit4] > "name":".auto_add_replicas",
[junit4] > "event":"nodeLost",
[junit4] > "waitFor":30,
[junit4] > "actions":[
[junit4] > {
[junit4] > "name":"auto_add_replicas_plan",
[junit4] > "class":"solr.AutoAddReplicasPlanAction"},
[junit4] > {
[junit4] > "name":"execute_plan",
[junit4] > "class":"solr.ExecutePlanAction"}],
[junit4] > "enabled":true}},
[junit4] > "listeners":{".auto_add_replicas.system":{
[junit4] > "trigger":".auto_add_replicas",
[junit4] > "afterAction":[],
[junit4] > "stage":[
[junit4] > "STARTED",
[junit4] > "ABORTED",
[junit4] > "SUCCEEDED",
[junit4] > "FAILED",
[junit4] > "BEFORE_ACTION",
[junit4] > "AFTER_ACTION",
[junit4] > "IGNORED"],
[junit4] > "class":"org.apache.
[...truncated too long message...]
c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@31e808ef: rootName = solr_51932, domain = solr.core..system.shard1.replica_n1, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@4333b179
[junit4] 2> 999179 INFO (coreCloseExecutor-3909-thread-1) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [.system_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@7a68e7c4
[junit4] 2> 999179 INFO (coreCloseExecutor-3909-thread-1) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_n2, tag=2053695428
[junit4] 2> 999180 INFO (coreCloseExecutor-3909-thread-1) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2dfdefe2: rootName = solr_51934, domain = solr.core..system.shard1.replica_n2, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@4de06ce2
[junit4] 2> 999180 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 999180 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 999181 INFO (zkCallback-1672-thread-2-processing-n:127.0.0.1:51964_solr) [n:127.0.0.1:51964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 999192 INFO (jetty-closer-1627-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 999192 INFO (jetty-closer-1627-thread-3) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@ab75de6: rootName = solr_51964, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@132d6c0e
[junit4] 2> 999194 INFO (jetty-closer-1627-thread-3) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:51964_solr
[junit4] 2> 999196 INFO (zkCallback-1650-thread-2-processing-n:127.0.0.1:51932_solr) [n:127.0.0.1:51932_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 999196 INFO (zkCallback-1648-thread-2-processing-n:127.0.0.1:51934_solr) [n:127.0.0.1:51934_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 999197 INFO (jetty-closer-1627-thread-3) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 999197 INFO (zkCallback-1672-thread-2-processing-n:127.0.0.1:51964_solr) [n:127.0.0.1:51964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 999198 INFO (jetty-closer-1627-thread-3) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 999201 INFO (coreCloseExecutor-3908-thread-1) [n:127.0.0.1:51932_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=1602002519
[junit4] 2> 999201 INFO (coreCloseExecutor-3909-thread-1) [n:127.0.0.1:51934_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=2053695428
[junit4] 2> 999203 INFO (jetty-closer-1627-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@62f99c7a{/solr,null,UNAVAILABLE}
[junit4] 2> 999205 INFO (jetty-closer-1627-thread-2) [ ] o.a.s.c.Overseer Overseer (id=99348462769012743-127.0.0.1:51934_solr-n_0000000000) closing
[junit4] 2> 999205 INFO (jetty-closer-1627-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@16235325{/solr,null,UNAVAILABLE}
[junit4] 2> 999205 INFO (OverseerStateUpdate-99348462769012743-127.0.0.1:51934_solr-n_0000000000) [n:127.0.0.1:51934_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:51934_solr
[junit4] 2> 999206 WARN (OverseerAutoScalingTriggerThread-99348462769012743-127.0.0.1:51934_solr-n_0000000000) [n:127.0.0.1:51934_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 999208 INFO (jetty-closer-1627-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@48883423{/solr,null,UNAVAILABLE}
[junit4] 2> 999209 ERROR (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 999209 INFO (SUITE-TestClusterStateProvider-seed#[C946F8ADA3128640]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51930 51930
[junit4] 2> 999224 INFO (Thread-1658) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51930 51930
[junit4] 2> 999313 WARN (Thread-1658) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 3 /solr/configs/.system
[junit4] 2> 3 /solr/configs/.system/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 20 /solr/collections/.system/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/99348462769012744-127.0.0.1:51932_solr-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /Users/jenkins/workspace/Lucene-Solr-7.x-MacOSX/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_C946F8ADA3128640-001
[junit4] 2> Jan 14, 2018 1:40:42 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1721, maxMBSortInHeap=6.58431214183433, sim=RandomSimilarity(queryNorm=false): {}, locale=ar-LB, timezone=Etc/GMT+8
[junit4] 2> NOTE: Mac OS X 10.11.6 x86_64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=3,threads=1,free=182461304,total=518979584
[junit4] 2> NOTE: All tests run in this JVM: [TestQueryWrapperFilter, TestInitParams, DataDrivenBlockJoinTest, SecurityConfHandlerTest, TestHttpShardHandlerFactory, TestTolerantUpdateProcessorRandomCloud, DirectUpdateHandlerOptimizeTest, BinaryUpdateRequestHandlerTest, TestSlowCompositeReaderWrapper, TestDistributedStatsComponentCardinality, DateRangeFieldTest, SignatureUpdateProcessorFactoryTest, SuggesterTSTTest, CursorPagingTest, TestImplicitCoreProperties, AutoScalingHandlerTest, TestRetrieveFieldsOptimizer, XmlUpdateRequestHandlerTest, TestFilteredDocIdSet, DeleteNodeTest, HighlighterConfigTest, LeaderInitiatedRecoveryOnShardRestartTest, CircularListTest, TestInPlaceUpdatesDistrib, XsltUpdateRequestHandlerTest, TestObjectReleaseTracker, OverseerModifyCollectionTest, WrapperMergePolicyFactoryTest, RAMDirectoryFactoryTest, TestPseudoReturnFields, NumericFieldsTest, HdfsCollectionsAPIDistributedZkTest, MetricsHandlerTest, TestComplexPhraseLeadingWildcard, TestRecoveryHdfs, DistributedTermsComponentTest, BlockJoinFacetDistribTest, CSVRequestHandlerTest, ConfigSetsAPITest, TestQuerySenderNoQuery, CoreMergeIndexesAdminHandlerTest, TestHalfAndHalfDocValues, TestFiltering, DistributedVersionInfoTest, NoCacheHeaderTest, TestCoreDiscovery, TestSweetSpotSimilarityFactory, TestDocTermOrds, SmileWriterTest, TestCloudJSONFacetJoinDomain, SpellCheckComponentTest, TestOmitPositions, TestUninvertingReader, ShardRoutingTest, HdfsThreadLeakTest, TestUtils, TestRecovery, URLClassifyProcessorTest, TestCopyFieldCollectionResource, ClassificationUpdateProcessorIntegrationTest, PathHierarchyTokenizerFactoryTest, DistributedFacetPivotLongTailTest, UniqFieldsUpdateProcessorFactoryTest, AnalyticsQueryTest, SolrJmxReporterTest, DistributedFacetExistsSmallTest, DistributedSuggestComponentTest, TestStreamBody, UnloadDistributedZkTest, V2StandaloneTest, HttpSolrCallGetCoreTest, ScriptEngineTest, TestTolerantSearch, CollectionTooManyReplicasTest, TestDefaultStatsCache, TestMissingGroups, TestQuerySenderListener, TestUseDocValuesAsStored2, SolrMetricManagerTest, TestLRUStatsCache, TestStressInPlaceUpdates, TestCollectionAPI, ShowFileRequestHandlerTest, TestPullReplicaErrorHandling, SolrInfoBeanTest, TestRequestStatusCollectionAPI, TestBackupRepositoryFactory, TestHdfsBackupRestoreCore, TestCloudNestedDocsSort, TestSolrCoreParser, DistributedQueryComponentCustomSortTest, StatsComponentTest, TestDeleteCollectionOnDownNodes, SortByFunctionTest, QueryParsingTest, TestCustomStream, SliceStateTest, TestLocalFSCloudBackupRestore, DisMaxRequestHandlerTest, TestRandomRequestDistribution, TestConfigSets, SolrIndexMetricsTest, TestClusterStateProvider]
[junit4] Completed [188/766 (1!)] on J0 in 6.13s, 2 tests, 1 failure <<< FAILURES!
[...truncated 50099 lines...]