You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/09/09 11:48:54 UTC
[JENKINS] Lucene-Solr-Tests-master - Build # 2793 - Still Unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2793/
2 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster.testSearchRate
Error Message:
java.util.concurrent.TimeoutException: last state: DocCollection(.system//clusterstate.json/385)={ "replicationFactor":"1", "pullReplicas":"0", "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"1", "tlogReplicas":"0", "autoCreated":"true", "shards":{"shard1":{ "replicas":{ "core_node2":{ "core":".system_shard1_replica_n1", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10055_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node3":{ "core":".system_shard1_replica_n1", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10098_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"80000000-7fffffff", "state":"active"}}}
Stack Trace:
java.io.IOException: java.util.concurrent.TimeoutException: last state: DocCollection(.system//clusterstate.json/385)={
"replicationFactor":"1",
"pullReplicas":"0",
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0",
"autoCreated":"true",
"shards":{"shard1":{
"replicas":{
"core_node2":{
"core":".system_shard1_replica_n1",
"leader":"true",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10055_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node3":{
"core":".system_shard1_replica_n1",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10098_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0}},
"range":"80000000-7fffffff",
"state":"active"}}}
at __randomizedtesting.SeedInfo.seed([BE751441B679E0C8:E33D0AC879BF4687]:0)
at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.createSystemCollection(SimClusterStateProvider.java:1226)
at org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster.setupTest(TestSimLargeCluster.java:137)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:969)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: last state: DocCollection(.system//clusterstate.json/385)={
"replicationFactor":"1",
"pullReplicas":"0",
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"1",
"tlogReplicas":"0",
"autoCreated":"true",
"shards":{"shard1":{
"replicas":{
"core_node2":{
"core":".system_shard1_replica_n1",
"leader":"true",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10055_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node3":{
"core":".system_shard1_replica_n1",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10098_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0}},
"range":"80000000-7fffffff",
"state":"active"}}}
at org.apache.solr.cloud.CloudTestUtils.waitForState(CloudTestUtils.java:112)
at org.apache.solr.cloud.autoscaling.sim.SimClusterStateProvider.createSystemCollection(SimClusterStateProvider.java:1223)
... 39 more
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction.testNodeAdded
Error Message:
ComputePlanAction should have computed exactly 1 operation, but was: [org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@74b6012b, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@16d32621] expected:<1> but was:<2>
Stack Trace:
java.lang.AssertionError: ComputePlanAction should have computed exactly 1 operation, but was: [org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@74b6012b, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@16d32621] expected:<1> but was:<2>
at __randomizedtesting.SeedInfo.seed([BE751441B679E0C8:DBB6423614DA48CB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction.testNodeAdded(TestSimComputePlanAction.java:314)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1742)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:935)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:971)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:985)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:944)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:830)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:891)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12502 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestSimComputePlanAction_BE751441B679E0C8-001/init-core-data-001
[junit4] 2> 183680 WARN (SUITE-TestSimComputePlanAction-seed#[BE751441B679E0C8]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 183680 INFO (SUITE-TestSimComputePlanAction-seed#[BE751441B679E0C8]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 183682 INFO (SUITE-TestSimComputePlanAction-seed#[BE751441B679E0C8]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 183682 INFO (SUITE-TestSimComputePlanAction-seed#[BE751441B679E0C8]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 183706 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 183706 DEBUG (SUITE-TestSimComputePlanAction-seed#[BE751441B679E0C8]-worker) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10000_solr
[junit4] 2> 183707 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
[junit4] 2> 183707 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 183707 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 183713 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 183716 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 183716 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
[junit4] 2> 183716 DEBUG (ScheduledTrigger-550-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183718 INFO (SUITE-TestSimComputePlanAction-seed#[BE751441B679E0C8]-worker) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 183721 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLost
[junit4] 2> 183721 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 183721 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 183722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 183723 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 183723 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 183723 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 183723 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 183723 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 183723 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 183723 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1220) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:177) [java/:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 183724 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 183724 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 183724 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 183725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 183725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 183725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 183725 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 183726 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 183726 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 183726 DEBUG (ScheduledTrigger-555-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183746 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183766 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183787 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183807 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183828 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183848 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183853 DEBUG (simCloudManagerPool-554-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 183853 DEBUG (simCloudManagerPool-554-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 183854 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 183855 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 183855 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 183861 DEBUG (simCloudManagerPool-554-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 183861 DEBUG (simCloudManagerPool-554-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 183862 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 183862 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 183862 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 183868 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183888 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183909 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183929 DEBUG (ScheduledTrigger-555-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183959 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 183979 DEBUG (ScheduledTrigger-555-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 183994 DEBUG (simCloudManagerPool-554-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 183994 DEBUG (simCloudManagerPool-554-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
[junit4] 2> 183994 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 183995 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger - Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 183995 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 183995 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
[junit4] 2> 183995 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 183999 DEBUG (ScheduledTrigger-555-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184003 DEBUG (simCloudManagerPool-554-thread-4) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeLost
[junit4] 2> 184015 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184019 DEBUG (ScheduledTrigger-555-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184022 DEBUG (simCloudManagerPool-554-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 184024 DEBUG (simCloudManagerPool-554-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 184039 DEBUG (ScheduledTrigger-555-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184040 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184046 INFO (simCloudManagerPool-554-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 3778771447 sessionWrapper.createTime 3778770359179074, this.sessionWrapper.createTime 3778770359179074
[junit4] 2> 184046 DEBUG (simCloudManagerPool-554-thread-4) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 184059 DEBUG (ScheduledTrigger-555-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184060 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184064 DEBUG (simCloudManagerPool-554-thread-5) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeLost / shard1
[junit4] 2> 184064 DEBUG (simCloudManagerPool-554-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeLost / shard1
[junit4] 2> 184066 DEBUG (simCloudManagerPool-554-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: core_node1
[junit4] 2> 184074 DEBUG (simCloudManagerPool-554-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeLost / shard1
[junit4] 2> 184075 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -- cluster state: znodeVersion: 6
[junit4] 2> live nodes:[127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> collections:{testNodeLost=DocCollection(testNodeLost//clusterstate.json/5)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}}
[junit4] 2> 184076 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeLost / shard1
[junit4] 2> 184076 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction Stopped_node : 127.0.0.1:10001_solr
[junit4] 2> 184076 DEBUG (simCloudManagerPool-554-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeLost / shard1
[junit4] 2> 184077 INFO (simCloudManagerPool-554-thread-8) [ ] o.a.s.c.ActionThrottle Throttling leader attempts - waiting for 4449ms
[junit4] 2> 184079 DEBUG (ScheduledTrigger-555-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184079 DEBUG (ScheduledTrigger-555-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 184080 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184080 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 184099 DEBUG (ScheduledTrigger-555-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184100 DEBUG (ScheduledTrigger-555-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184120 DEBUG (ScheduledTrigger-555-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184120 DEBUG (ScheduledTrigger-555-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184140 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184141 DEBUG (ScheduledTrigger-555-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184160 DEBUG (ScheduledTrigger-555-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184161 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184166 DEBUG (simCloudManagerPool-554-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeLost / shard1: core_node2
[junit4] 2> 184180 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184181 DEBUG (ScheduledTrigger-555-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184201 DEBUG (ScheduledTrigger-555-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184201 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184231 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2
[junit4] 2> 184231 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10001_solr]
[junit4] 2> 184233 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"d6cc62dd5d2e8Tdse9nldtctsz89wkpxmokx8tq",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":3778773125550824,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[3778773125550824],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 184233 DEBUG (ScheduledTrigger-555-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance, node_lost_trigger]
[junit4] 2> 184241 DEBUG (simCloudManagerPool-554-thread-9) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 184252 DEBUG (simCloudManagerPool-554-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 184254 DEBUG (simCloudManagerPool-554-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 184255 INFO (simCloudManagerPool-554-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 3778781904 sessionWrapper.createTime 3778781861155074, this.sessionWrapper.createTime 3778781861155074
[junit4] 2> 184255 DEBUG (simCloudManagerPool-554-thread-9) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 184266 DEBUG (simCloudManagerPool-554-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 184276 DEBUG (simCloudManagerPool-554-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 184276 DEBUG (simCloudManagerPool-554-thread-11) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node1
[junit4] 2> 184281 DEBUG (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"d6cc62dd5d2e8Tdse9nldtctsz89wkpxmokx8tq",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":3778773125550824,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[3778773125550824],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":3778780848645524,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}}
[junit4] 2> 184283 DEBUG (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"d6cc62dd5d2e8Tdse9nldtctsz89wkpxmokx8tq",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":3778773125550824,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[3778773125550824],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":3778780848645524,
[junit4] 2> "nodeNames":["127.0.0.1:10001_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
[junit4] 2> 184285 DEBUG (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 184287 DEBUG (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 184299 INFO (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeLost&targetNode=127.0.0.1:10002_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 184300 INFO (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 3778784155 sessionWrapper.createTime 3778783499665474, this.sessionWrapper.createTime 3778783499665474
[junit4] 2> 184300 DEBUG (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 184311 DEBUG (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 184311 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -------------_ FINAL STATE --------------
[junit4] 2> 184312 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Node values: {
[junit4] 2> "127.0.0.1:10001_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1023,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10001_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10001,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10000_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":999,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10000_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10000,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10002_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":1,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1023,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10002_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10002,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
[junit4] 2> 184312 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Live nodes: [127.0.0.1:10000_solr]
[junit4] 2> 184312 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection .system state: DocCollection(.system//clusterstate.json/13)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 184312 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection testNodeLost state: DocCollection(testNodeLost//clusterstate.json/13)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10001_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeLost_shard1_replica_n2",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 184312 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLost
[junit4] 2> 184313 DEBUG (simCloudManagerPool-554-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 184322 WARN (simCloudManagerPool-554-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider Can't find any active replicas for .system / shard1: shard1:{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}
[junit4] 2> 184322 DEBUG (simCloudManagerPool-554-thread-13) [ ] o.a.s.c.a.s.SimClusterStateProvider -- liveNodes: [127.0.0.1:10000_solr]
[junit4] 2> 184325 INFO (TEST-TestSimComputePlanAction.testNodeLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 1
[junit4] 2> ## Empty nodes: 0
[junit4] 2> ## Dead nodes: 2
[junit4] 2> ## 127.0.0.1:10001_solr
[junit4] 2> ## 127.0.0.1:10002_solr
[junit4] 2> ## Collections: [.system, testNodeLost]
[junit4] 2> ## Max replicas per node: 1
[junit4] 2> ## Min replicas per node: 1
[junit4] 2> ## Total replicas: 1
[junit4] 2> ## * testNodeLost 1
[junit4] 2> ## - active 1
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - autoscaling 3
[junit4] 2> ## - update 2
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: node_lost_trigger
[junit4] 2> ## - BEFORE_ACTION 1
[junit4] 2> ## - STARTED 1
[junit4] 2>
[junit4] 2> 184332 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAdded
[junit4] 2> 184333 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
[junit4] 2> 184333 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
[junit4] 2> 184333 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 184333 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- deleted /autoscaling/nodeLost/127.0.0.1:10002_solr
[junit4] 2> 184334 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184334 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
[junit4] 2> 184335 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10001_solr
[junit4] 2> 184335 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10002_solr
[junit4] 2> 184335 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 184335 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2048) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:189) [java/:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 184335 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 184335 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 184335 WARN (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.a.SystemLogListener Exception sending event to collection .system
[junit4] 2> java.io.IOException: java.lang.InterruptedException
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:632) ~[test/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.SystemLogListener.onEvent(SystemLogListener.java:118) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:792) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerListeners.fireListeners(ScheduledTriggers.java:768) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:315) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> Caused by: java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:404) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.get(FutureTask.java:191) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.cloud.autoscaling.sim.SimCloudManager.request(SimCloudManager.java:630) ~[test/:?]
[junit4] 2> ... 10 more
[junit4] 2> 184337 DEBUG (simCloudManagerPool-554-thread-14) [ ] o.a.s.c.a.s.SimClusterStateProvider -- no leader in shard1:{
[junit4] 2> "replicas":{"core_node1":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "SEARCHER.searcher.maxDoc":2,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":11240,
[junit4] 2> "node_name":"127.0.0.1:10002_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.0468065738677979E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":2}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}
[junit4] 2> 184337 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 184337 DEBUG (simCloudManagerPool-554-thread-15) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: .system
[junit4] 2> 184359 DEBUG (simCloudManagerPool-554-thread-16) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for .system / shard1
[junit4] 2> 184359 DEBUG (simCloudManagerPool-554-thread-17) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for .system / shard1
[junit4] 2> 184359 DEBUG (simCloudManagerPool-554-thread-17) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for .system / shard1: core_node2
[junit4] 2> 184360 WARN (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Unhandled exception executing actions
[junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_lost_trigger has been closed.
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:548) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:318) ~[java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_172]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_172]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) ~[java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_172]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_172]
[junit4] 2> at java.lang.Thread.run(Thread.java:748) [?:1.8.0_172]
[junit4] 2> 184363 DEBUG (AutoscalingActionExecutor-556-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 4076 ms for event id=d6cc62dd5d2e8Tdse9nldtctsz89wkpxmokx8tq
[junit4] 2> 184363 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 184363 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 184363 DEBUG (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 184365 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 184365 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
[junit4] 2> 184366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion -1
[junit4] 2> 184366 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
[junit4] 2> 184367 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 184367 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184367 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
[junit4] 2> 184367 DEBUG (ScheduledTrigger-558-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184387 DEBUG (ScheduledTrigger-558-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184407 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184428 DEBUG (ScheduledTrigger-558-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184448 DEBUG (ScheduledTrigger-558-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184465 DEBUG (simCloudManagerPool-557-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184465 DEBUG (simCloudManagerPool-557-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
[junit4] 2> 184466 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
[junit4] 2> 184466 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184466 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
[junit4] 2> 184468 DEBUG (simCloudManagerPool-557-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184468 DEBUG (simCloudManagerPool-557-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
[junit4] 2> 184468 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184468 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
[junit4] 2> 184468 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184468 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
[junit4] 2> 184488 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184509 DEBUG (ScheduledTrigger-558-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184529 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184549 DEBUG (ScheduledTrigger-558-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184569 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184574 DEBUG (simCloudManagerPool-557-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184574 DEBUG (simCloudManagerPool-557-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
[junit4] 2> 184575 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
[junit4] 2> 184575 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 184575 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction$AssertingTriggerAction}], enabled=true}
[junit4] 2> 184576 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
[junit4] 2> 184576 DEBUG (ScheduledTrigger-558-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 184576 DEBUG (ScheduledTrigger-558-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 184577 DEBUG (simCloudManagerPool-557-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184577 DEBUG (simCloudManagerPool-557-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
[junit4] 2> 184578 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
[junit4] 2> 184578 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
[junit4] 2> 184579 DEBUG (simCloudManagerPool-557-thread-5) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeAdded
[junit4] 2> 184589 WARN (simCloudManagerPool-557-thread-5) [ ] o.a.s.c.a.c.CreateCollectionCmd Specified number of replicas of 4 on collection testNodeAdded is higher than the number of Solr instances currently live or live and part of your createNodeSet(1). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 184590 DEBUG (ScheduledTrigger-558-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184590 DEBUG (simCloudManagerPool-557-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 184590 DEBUG (simCloudManagerPool-557-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 184591 INFO (simCloudManagerPool-557-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 3778798727 sessionWrapper.createTime 3778798666502324, this.sessionWrapper.createTime 3778798666502324
[junit4] 2> 184591 DEBUG (simCloudManagerPool-557-thread-5) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 184597 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 184597 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 184602 DEBUG (simCloudManagerPool-557-thread-6) [ ] o.a.s.c.a.s.SimClusterStateProvider -- submit leader election for testNodeAdded / shard1
[junit4] 2> 184602 DEBUG (simCloudManagerPool-557-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider Running leader election for testNodeAdded / shard1
[junit4] 2> 184603 DEBUG (simCloudManagerPool-557-thread-10) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard1: core_node1
[junit4] 2> 184610 DEBUG (ScheduledTrigger-558-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184613 DEBUG (simCloudManagerPool-557-thread-7) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeAdded / shard1
[junit4] 2> 184617 DEBUG (ScheduledTrigger-558-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 184617 DEBUG (ScheduledTrigger-558-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 184623 DEBUG (simCloudManagerPool-557-thread-8) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeAdded / shard1
[junit4] 2> 184626 DEBUG (simCloudManagerPool-557-thread-11) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184626 DEBUG (simCloudManagerPool-557-thread-11) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
[junit4] 2> 184627 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
[junit4] 2> 184628 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
[junit4] 2> 184630 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184634 DEBUG (simCloudManagerPool-557-thread-9) [ ] o.a.s.c.a.s.SimClusterStateProvider -- already has leader testNodeAdded / shard1
[junit4] 2> 184637 DEBUG (ScheduledTrigger-558-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 184637 DEBUG (ScheduledTrigger-558-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 184637 DEBUG (ScheduledTrigger-558-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 3778801029810924
[junit4] 2> 184650 DEBUG (ScheduledTrigger-558-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 184658 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 184658 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 184658 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [3778801029810924], now=3778802048214774
[junit4] 2> 184658 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"d6cccad0f0aecTdse9nldtctsz89wkpxmokx8tv",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":3778801029810924,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[3778801029810924],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 184658 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, .scheduled_maintenance, node_added_trigger]
[junit4] 2> 184660 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"d6cccad0f0aecTdse9nldtctsz89wkpxmokx8tv",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":3778801029810924,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[3778801029810924],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":3778802081354674,
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 184660 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"d6cccad0f0aecTdse9nldtctsz89wkpxmokx8tv",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":3778801029810924,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[3778801029810924],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":3778802081354674,
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
[junit4] 2> 184660 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper Creating a new session
[junit4] 2> 184660 DEBUG (ScheduledTrigger-558-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - removing marker path: /autoscaling/nodeAdded/127.0.0.1:10003_solr
[junit4] 2> 184662 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper New session created
[junit4] 2> 184664 INFO (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeAdded&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node1
[junit4] 2> 184664 INFO (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeAdded&targetNode=127.0.0.1:10003_solr&inPlaceMove=true&replica=core_node2
[junit4] 2> 184666 INFO (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 3778802445 sessionWrapper.createTime 3778802248022724, this.sessionWrapper.createTime 3778802248022724
[junit4] 2> 184666 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper session set to NULL
[junit4] 2> 184675 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction Node values: {
[junit4] 2> "127.0.0.1:10000_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":5,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":995,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10000_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10000,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10003_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":0,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1024,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10003_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10003,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
[junit4] 2> 184676 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction Live nodes: [127.0.0.1:10000_solr, 127.0.0.1:10003_solr], collection state: DocCollection(testNodeAdded//clusterstate.json/29)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"4",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n3",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n4",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 184676 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction -------------_ FINAL STATE --------------
[junit4] 2> 184676 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Node values: {
[junit4] 2> "127.0.0.1:10000_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":5,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":995,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10000_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10000,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0},
[junit4] 2> "127.0.0.1:10003_solr":{
[junit4] 2> "sysprop.java.vendor":"Oracle Corporation",
[junit4] 2> "cores":0,
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "freedisk":1024,
[junit4] 2> "sysprop.java.version":"1.8.0_172",
[junit4] 2> "ip_1":"127",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"1",
[junit4] 2> "node":"127.0.0.1:10003_solr",
[junit4] 2> "metrics:solr.jvm:buffers.direct.Count":0,
[junit4] 2> "port":10003,
[junit4] 2> "heapUsage":123450000,
[junit4] 2> "sysLoadAvg":1.0,
[junit4] 2> "metrics:solr.node:ADMIN./admin/authorization.clientErrors:count":0}}
[junit4] 2> 184676 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Live nodes: [127.0.0.1:10000_solr, 127.0.0.1:10003_solr]
[junit4] 2> 184676 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection .system state: DocCollection(.system//clusterstate.json/31)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{"core_node2":{
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":4,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":12240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":1.1399388313293457E-5,
[junit4] 2> "SEARCHER.searcher.numDocs":4}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 184676 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.TestSimComputePlanAction * Collection testNodeAdded state: DocCollection(testNodeAdded//clusterstate.json/31)={
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"4",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "autoCreated":"true",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "replicas":{
[junit4] 2> "core_node1":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node2":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n3",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n4",
[junit4] 2> "SEARCHER.searcher.maxDoc":0,
[junit4] 2> "SEARCHER.searcher.deletedDocs":0,
[junit4] 2> "INDEX.sizeInBytes":10240,
[junit4] 2> "node_name":"127.0.0.1:10000_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInGB":9.5367431640625E-6,
[junit4] 2> "SEARCHER.searcher.numDocs":0}},
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active"}}}
[junit4] 2> 184677 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAdded
[junit4] 2> 184677 INFO (TEST-TestSimComputePlanAction.testNodeAdded-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #######################################
[junit4] 2> ############ CLUSTER STATE ############
[junit4] 2> #######################################
[junit4] 2> ## Live nodes: 2
[junit4] 2> ## Empty nodes: 1
[junit4] 2> ## Dead nodes: 0
[junit4] 2> ## Collections: [.system, testNodeAdded]
[junit4] 2> ## Max replicas per node: 5
[junit4] 2> ## Min replicas per node: 0
[junit4] 2> ## Total replicas: 5
[junit4] 2> ## * .system 1
[junit4] 2> ## - active 1
[junit4] 2> ## * testNodeAdded 4
[junit4] 2> ## - active 4
[junit4] 2> ######### Solr op counts ##########
[junit4] 2> ## - CREATE 1
[junit4] 2> ## - autoscaling 5
[junit4] 2> ## - update 3
[junit4] 2> ######### Autoscaling event counts ###########
[junit4] 2> ## * Trigger: node_added_trigger
[junit4] 2> ## - AFTER_ACTION 1
[junit4] 2> ## - BEFORE_ACTION 1
[junit4] 2> ## - STARTED 1
[junit4] 2> ## * Trigger: node_lost_trigger
[junit4] 2> ## - AFTER_ACTION 1
[junit4] 2> ## - FAILED 1
[junit4] 2>
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSimComputePlanAction -Dtests.method=testNodeAdded -Dtests.seed=BE751441B679E0C8 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=fr-FR -Dtests.timezone=Africa/Niamey -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] FAILURE 0.36s J0 | TestSimComputePlanAction.testNodeAdded <<<
[junit4] > Throwable #1: java.lang.AssertionError: ComputePlanAction should have computed exactly 1 operation, but was: [org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@74b6012b, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@16d32621] expected:<1> but was:<2>
[junit4] > at __randomizedtesting.SeedInfo.seed([BE751441B679E0C8:DBB6423614DA48CB]:0)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.TestSimComputePlanAction.testNodeAdded(TestSimComputePlanAction.java:314)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 184694 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 184694 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .scheduled_maintenance after 100ms
[junit4] 2> 184694 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 100ms
[junit4] 2> 184694 DEBUG (AutoscalingActionExecutor-559-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 1714 ms for event id=d6cccad0f0aecTdse9nldtctsz89wkpxmokx8tv
[junit4] 2> 184697 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeWithMultipleReplicasLost
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
[junit4] 2> 184698 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10003_solr
[junit4] 2> 184698 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 184698 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimNodeStateProvider --removing value for 127.0.0.1:10000_solr
[junit4] 2> 184698 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
[junit4] 2> 184698 INFO (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down scheduled thread pool executor now
[junit4] 2> 184698 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Shutting down action executor now
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for action executor
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers Awaiting termination for scheduled thread pool executor
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.ScheduledTriggers ScheduledTriggers closed completely
[junit4] 2> 184698 DEBUG (TEST-TestSimComputePlanAction.testNodeWithMultipleReplicasLost-seed#[BE751441B679E0C8]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 184700 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 184700 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14
[junit4] 2> 184700 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion -1
[junit4] 2> 184700 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
[junit4] 2> 184701 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10003_solr]
[junit4] 2> 184701 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184701 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
[junit4] 2> 184701 DEBUG (ScheduledTrigger-561-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184722 DEBUG (ScheduledTrigger-561-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184742 DEBUG (ScheduledTrigger-561-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184762 DEBUG (ScheduledTrigger-561-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184782 DEBUG (ScheduledTrigger-561-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184800 DEBUG (simCloudManagerPool-560-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184801 DEBUG (simCloudManagerPool-560-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15
[junit4] 2> 184801 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
[junit4] 2> 184802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184802 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
[junit4] 2> 184802 DEBUG (simCloudManagerPool-560-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184802 DEBUG (ScheduledTrigger-561-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184802 DEBUG (simCloudManagerPool-560-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 16
[junit4] 2> 184803 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 16
[junit4] 2> 184803 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 184803 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 16, lastZnodeVersion 16
[junit4] 2> 184823 DEBUG (ScheduledTrigger-561-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184843 DEBUG (ScheduledTrigger-561-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184863 DEBUG (ScheduledTrigger-561-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184883 DEBUG (ScheduledTrigger-561-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 184903 DEBUG (ScheduledTrigger-561-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 184904 DEBUG (simCloudManagerPool-560-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 184905 DEBUG (simCloudManagerPool-560-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 17
[junit4] 2> 184905 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 17
[junit4] 2> 184905 DEBUG (Simulated OverseerAutoScalingTriggerThread) [
[...truncated too long message...]
ed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null806333630
[copy] Copying 238 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null806333630
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null806333630
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 19ms :: artifacts dl 1ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 94 minutes 32 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-Tests-master - Build # 2795 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2795/
1 tests failed.
FAILED: org.apache.solr.cloud.cdcr.CdcrBidirectionalTest.testBiDir
Error Message:
Captured an uncaught exception in thread: Thread[id=3004, name=cdcr-replicator-1478-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3004, name=cdcr-replicator-1478-thread-1, state=RUNNABLE, group=TGRP-CdcrBidirectionalTest]
Caused by: java.lang.AssertionError: 1611158363993800704 != 1611158363873214464
at __randomizedtesting.SeedInfo.seed([AE806A5A97EFF027]:0)
at org.apache.solr.update.CdcrUpdateLog$CdcrLogReader.forwardSeek(CdcrUpdateLog.java:611)
at org.apache.solr.handler.CdcrReplicator.run(CdcrReplicator.java:125)
at org.apache.solr.handler.CdcrReplicatorScheduler.lambda$null$0(CdcrReplicatorScheduler.java:81)
at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12791 lines...]
[junit4] Suite: org.apache.solr.cloud.cdcr.CdcrBidirectionalTest
[junit4] 2> 253503 INFO (SUITE-CdcrBidirectionalTest-seed#[AE806A5A97EFF027]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/init-core-data-001
[junit4] 2> 253503 WARN (SUITE-CdcrBidirectionalTest-seed#[AE806A5A97EFF027]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 253503 INFO (SUITE-CdcrBidirectionalTest-seed#[AE806A5A97EFF027]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 253505 INFO (SUITE-CdcrBidirectionalTest-seed#[AE806A5A97EFF027]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 253520 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBiDir
[junit4] 2> 253521 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster2-001
[junit4] 2> 253521 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 253521 INFO (Thread-486) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 253521 INFO (Thread-486) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 253523 ERROR (Thread-486) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 253621 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.ZkTestServer start zk server on port:39460
[junit4] 2> 253625 INFO (zkConnectionManagerCallback-864-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 253630 INFO (jetty-launcher-861-thread-1) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 253631 INFO (jetty-launcher-861-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 253631 INFO (jetty-launcher-861-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 253631 INFO (jetty-launcher-861-thread-1) [ ] o.e.j.s.session node0 Scavenging every 600000ms
[junit4] 2> 253631 INFO (jetty-launcher-861-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2ea7340d{/solr,null,AVAILABLE}
[junit4] 2> 253633 INFO (jetty-launcher-861-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4367b523{HTTP/1.1,[http/1.1]}{127.0.0.1:35584}
[junit4] 2> 253633 INFO (jetty-launcher-861-thread-1) [ ] o.e.j.s.Server Started @253675ms
[junit4] 2> 253633 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35584}
[junit4] 2> 253633 ERROR (jetty-launcher-861-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 253633 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 253633 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 253633 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 253634 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 253634 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-09-09T19:12:16.156Z
[junit4] 2> 253635 INFO (zkConnectionManagerCallback-866-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 253636 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 253785 INFO (jetty-launcher-861-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39460/solr
[junit4] 2> 253787 INFO (zkConnectionManagerCallback-870-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 253790 INFO (zkConnectionManagerCallback-872-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 253854 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35584_solr
[junit4] 2> 253855 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.c.Overseer Overseer (id=72307226829914115-127.0.0.1:35584_solr-n_0000000000) starting
[junit4] 2> 253861 INFO (zkConnectionManagerCallback-879-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 253862 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39460/solr ready
[junit4] 2> 253863 INFO (OverseerStateUpdate-72307226829914115-127.0.0.1:35584_solr-n_0000000000) [n:127.0.0.1:35584_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:35584_solr
[junit4] 2> 253863 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35584_solr
[junit4] 2> 253870 INFO (OverseerStateUpdate-72307226829914115-127.0.0.1:35584_solr-n_0000000000) [n:127.0.0.1:35584_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 253872 INFO (zkCallback-878-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 253883 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 253906 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35584.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 253913 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35584.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 253913 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35584.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 253914 INFO (jetty-launcher-861-thread-1) [n:127.0.0.1:35584_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster2-001/node1/.
[junit4] 2> 253931 INFO (zkConnectionManagerCallback-882-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 253934 INFO (zkConnectionManagerCallback-885-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 253935 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster1-001
[junit4] 2> 253936 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 253936 INFO (Thread-496) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 253936 INFO (Thread-496) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 253938 ERROR (Thread-496) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 254036 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.ZkTestServer start zk server on port:38401
[junit4] 2> 254039 INFO (zkConnectionManagerCallback-889-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254043 INFO (jetty-launcher-886-thread-1) [ ] o.e.j.s.Server jetty-9.4.11.v20180605; built: 2018-06-05T18:24:03.829Z; git: d5fc0523cfa96bfebfbda19606cad384d772f04c; jvm 1.8.0_172-b11
[junit4] 2> 254044 INFO (jetty-launcher-886-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 254044 INFO (jetty-launcher-886-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 254044 INFO (jetty-launcher-886-thread-1) [ ] o.e.j.s.session node0 Scavenging every 660000ms
[junit4] 2> 254045 INFO (jetty-launcher-886-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@493a2175{/solr,null,AVAILABLE}
[junit4] 2> 254046 INFO (jetty-launcher-886-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2b360816{HTTP/1.1,[http/1.1]}{127.0.0.1:45524}
[junit4] 2> 254046 INFO (jetty-launcher-886-thread-1) [ ] o.e.j.s.Server Started @254088ms
[junit4] 2> 254046 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45524}
[junit4] 2> 254046 ERROR (jetty-launcher-886-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 254047 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.s.SolrDispatchFilter Using logger factory org.apache.logging.slf4j.Log4jLoggerFactory
[junit4] 2> 254047 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 254047 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 254047 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 254047 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-09-09T19:12:16.569Z
[junit4] 2> 254049 INFO (zkConnectionManagerCallback-891-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254050 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 254117 INFO (jetty-launcher-886-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:38401/solr
[junit4] 2> 254118 INFO (zkConnectionManagerCallback-895-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254120 INFO (zkConnectionManagerCallback-897-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254197 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45524_solr
[junit4] 2> 254198 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.c.Overseer Overseer (id=72307226857111555-127.0.0.1:45524_solr-n_0000000000) starting
[junit4] 2> 254212 INFO (zkConnectionManagerCallback-904-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254214 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38401/solr ready
[junit4] 2> 254215 INFO (OverseerStateUpdate-72307226857111555-127.0.0.1:45524_solr-n_0000000000) [n:127.0.0.1:45524_solr ] o.a.s.c.Overseer Starting to work on the main queue : 127.0.0.1:45524_solr
[junit4] 2> 254215 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45524_solr
[junit4] 2> 254238 INFO (zkCallback-903-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 254238 INFO (zkCallback-896-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 254248 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 254271 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45524.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 254286 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45524.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 254286 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45524.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 254287 INFO (jetty-launcher-886-thread-1) [n:127.0.0.1:45524_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster1-001/node1/.
[junit4] 2> 254326 INFO (zkConnectionManagerCallback-907-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254331 INFO (zkConnectionManagerCallback-910-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254333 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.c.CdcrBidirectionalTest cluster2 zkHost = 127.0.0.1:39460/solr
[junit4] 2> 254333 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.c.CdcrBidirectionalTest cluster1 zkHost = 127.0.0.1:38401/solr
[junit4] 2> 254335 INFO (zkConnectionManagerCallback-912-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254360 INFO (zkConnectionManagerCallback-916-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 254361 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 254362 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:38401/solr ready
[junit4] 2> 254371 INFO (qtp1391551120-2860) [n:127.0.0.1:45524_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=cdcr-cluster1&maxShardsPerNode=2&name=cdcr-cluster1&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 254382 INFO (OverseerThreadFactory-1436-thread-1-processing-n:127.0.0.1:45524_solr) [n:127.0.0.1:45524_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection cdcr-cluster1
[junit4] 2> 254495 INFO (OverseerStateUpdate-72307226857111555-127.0.0.1:45524_solr-n_0000000000) [n:127.0.0.1:45524_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-cluster1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:45524/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 254515 INFO (OverseerStateUpdate-72307226857111555-127.0.0.1:45524_solr-n_0000000000) [n:127.0.0.1:45524_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster1",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"cdcr-cluster1_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:45524/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 254719 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster1_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 254719 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 254721 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr x:cdcr-cluster1_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster1_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 255741 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 255752 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 255755 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-cluster1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 255757 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 255757 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster1_shard1_replica_n1' using configuration from collection cdcr-cluster1, trusted=true
[junit4] 2> 255759 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.IndexSchema [cdcr-cluster1_shard2_replica_n2] Schema name=minimal
[junit4] 2> 255761 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 255761 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster1_shard2_replica_n2' using configuration from collection cdcr-cluster1, trusted=true
[junit4] 2> 255761 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45524.solr.core.cdcr-cluster1.shard2.replica_n2' (registry 'solr.core.cdcr-cluster1.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 255762 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45524.solr.core.cdcr-cluster1.shard1.replica_n1' (registry 'solr.core.cdcr-cluster1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 255768 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrCore [[cdcr-cluster1_shard2_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster1-001/node1/cdcr-cluster1_shard2_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster1-001/node1/./cdcr-cluster1_shard2_replica_n2/data/]
[junit4] 2> 255783 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-cluster1_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster1-001/node1/cdcr-cluster1_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster1-001/node1/./cdcr-cluster1_shard1_replica_n1/data/]
[junit4] 2> 255862 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 255862 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 255862 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 255863 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 255864 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 255864 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 255864 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 255864 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 255866 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7683a486[cdcr-cluster1_shard1_replica_n1] main]
[junit4] 2> 255866 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@3081a5a[cdcr-cluster1_shard2_replica_n2] main]
[junit4] 2> 255868 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster1
[junit4] 2> 255868 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster1
[junit4] 2> 255868 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster1
[junit4] 2> 255868 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster1/managed-schema
[junit4] 2> 255869 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster1
[junit4] 2> 255869 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 255869 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster1/managed-schema
[junit4] 2> 255869 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 255869 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 255870 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 255874 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-cluster1/cdcr/state/buffer
[junit4] 2> 255878 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-cluster1/cdcr/state/process
[junit4] 2> 255901 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1611158350382235648
[junit4] 2> 255902 INFO (searcherExecutor-1442-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrCore [cdcr-cluster1_shard2_replica_n2] Registered new searcher Searcher@3081a5a[cdcr-cluster1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 255902 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1611158350383284224
[junit4] 2> 255903 INFO (searcherExecutor-1441-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard1_replica_n1 c:cdcr-cluster1 s:shard1 r:core_node3) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-cluster1_shard1_replica_n1] Registered new searcher Searcher@7683a486[cdcr-cluster1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 255910 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster1/terms/shard1 to Terms{values={core_node3=0}, version=0}
[junit4] 2> 255911 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster1/terms/shard2 to Terms{values={core_node4=0}, version=0}
[junit4] 2> 255920 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 255920 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 255920 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45524/solr/cdcr-cluster1_shard1_replica_n1/
[junit4] 2> 255930 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 255930 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45524/solr/cdcr-cluster1_shard1_replica_n1/ has no replicas
[junit4] 2> 255930 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 255932 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 255932 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 255932 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45524/solr/cdcr-cluster1_shard2_replica_n2/
[junit4] 2> 255932 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 255933 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:45524/solr/cdcr-cluster1_shard2_replica_n2/ has no replicas
[junit4] 2> 255933 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 255933 INFO (zkCallback-896-thread-2) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster1:shard1
[junit4] 2> 255936 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45524/solr/cdcr-cluster1_shard1_replica_n1/ shard1
[junit4] 2> 255938 INFO (zkCallback-896-thread-2) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster1:shard2
[junit4] 2> 255940 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45524/solr/cdcr-cluster1_shard2_replica_n2/ shard2
[junit4] 2> 256041 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 256043 INFO (qtp1391551120-2855) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster1_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1324
[junit4] 2> 256088 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 256090 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster1&newCollection=true&collection=cdcr-cluster1&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster1_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1369
[junit4] 2> 256093 INFO (qtp1391551120-2860) [n:127.0.0.1:45524_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 256191 INFO (zkCallback-896-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster1/state.json] for collection [cdcr-cluster1] has occurred - updating... (live nodes size: [1])
[junit4] 2> 256380 INFO (OverseerCollectionConfigSetProcessor-72307226857111555-127.0.0.1:45524_solr-n_0000000000) [n:127.0.0.1:45524_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 257093 INFO (qtp1391551120-2860) [n:127.0.0.1:45524_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=cdcr-cluster1&maxShardsPerNode=2&name=cdcr-cluster1&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2722
[junit4] 2> 257096 INFO (zkConnectionManagerCallback-920-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 257101 INFO (zkConnectionManagerCallback-924-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 257102 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 257103 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39460/solr ready
[junit4] 2> 257105 INFO (qtp1299351791-2807) [n:127.0.0.1:35584_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params collection.configName=cdcr-cluster2&maxShardsPerNode=2&name=cdcr-cluster2&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 257108 INFO (OverseerThreadFactory-1423-thread-1-processing-n:127.0.0.1:35584_solr) [n:127.0.0.1:35584_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection cdcr-cluster2
[junit4] 2> 257215 INFO (OverseerStateUpdate-72307226829914115-127.0.0.1:35584_solr-n_0000000000) [n:127.0.0.1:35584_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster2",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"cdcr-cluster2_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:35584/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 257217 INFO (OverseerStateUpdate-72307226829914115-127.0.0.1:35584_solr-n_0000000000) [n:127.0.0.1:35584_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"cdcr-cluster2",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"cdcr-cluster2_shard2_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:35584/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 257428 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster2_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 257428 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 257438 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster2_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin
[junit4] 2> 258473 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 258478 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 258480 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.IndexSchema [cdcr-cluster2_shard2_replica_n2] Schema name=minimal
[junit4] 2> 258482 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 258482 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster2_shard2_replica_n2' using configuration from collection cdcr-cluster2, trusted=true
[junit4] 2> 258483 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35584.solr.core.cdcr-cluster2.shard2.replica_n2' (registry 'solr.core.cdcr-cluster2.shard2.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 258483 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrCore [[cdcr-cluster2_shard2_replica_n2] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster2-001/node1/cdcr-cluster2_shard2_replica_n2], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster2-001/node1/./cdcr-cluster2_shard2_replica_n2/data/]
[junit4] 2> 258488 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.IndexSchema [cdcr-cluster2_shard1_replica_n1] Schema name=minimal
[junit4] 2> 258490 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 258490 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'cdcr-cluster2_shard1_replica_n1' using configuration from collection cdcr-cluster2, trusted=true
[junit4] 2> 258490 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35584.solr.core.cdcr-cluster2.shard1.replica_n1' (registry 'solr.core.cdcr-cluster2.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2c7cf6b8
[junit4] 2> 258490 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrCore [[cdcr-cluster2_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster2-001/node1/cdcr-cluster2_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/build/solr-core/test/J2/temp/solr.cloud.cdcr.CdcrBidirectionalTest_AE806A5A97EFF027-001/cdcr-cluster2-001/node1/./cdcr-cluster2_shard1_replica_n1/data/]
[junit4] 2> 258535 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 258535 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 258536 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
[junit4] 2> 258536 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 258536 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 258536 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 258537 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 258537 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 258539 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@724a9774[cdcr-cluster2_shard2_replica_n2] main]
[junit4] 2> 258539 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@48c004db[cdcr-cluster2_shard1_replica_n1] main]
[junit4] 2> 258540 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster2
[junit4] 2> 258540 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/cdcr-cluster2
[junit4] 2> 258540 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster2
[junit4] 2> 258540 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster2/managed-schema
[junit4] 2> 258540 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/cdcr-cluster2
[junit4] 2> 258540 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Creating ZooKeeper watch for the managed schema at /configs/cdcr-cluster2/managed-schema
[junit4] 2> 258540 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 258541 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.ZkIndexSchemaReader Current schema version 0 is already the latest
[junit4] 2> 258541 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 258541 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 258546 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrBufferStateManager Created znode /collections/cdcr-cluster2/cdcr/state/buffer
[junit4] 2> 258547 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrProcessStateManager Created znode /collections/cdcr-cluster2/cdcr/state/process
[junit4] 2> 258548 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1611158353157816320
[junit4] 2> 258549 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1611158353158864896
[junit4] 2> 258549 INFO (searcherExecutor-1454-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-cluster2_shard1_replica_n1] Registered new searcher Searcher@48c004db[cdcr-cluster2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 258550 INFO (searcherExecutor-1453-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrCore [cdcr-cluster2_shard2_replica_n2] Registered new searcher Searcher@724a9774[cdcr-cluster2_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 258553 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster2/terms/shard2 to Terms{values={core_node4=0}, version=0}
[junit4] 2> 258554 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster2/terms/shard1 to Terms{values={core_node3=0}, version=0}
[junit4] 2> 258559 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 258559 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 258559 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35584/solr/cdcr-cluster2_shard2_replica_n2/
[junit4] 2> 258559 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 258559 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 258559 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35584/solr/cdcr-cluster2_shard1_replica_n1/
[junit4] 2> 258559 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 258559 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 258560 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SyncStrategy http://127.0.0.1:35584/solr/cdcr-cluster2_shard2_replica_n2/ has no replicas
[junit4] 2> 258560 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 258560 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:35584/solr/cdcr-cluster2_shard1_replica_n1/ has no replicas
[junit4] 2> 258560 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 258562 INFO (zkCallback-871-thread-1) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster2:shard2
[junit4] 2> 258562 INFO (zkCallback-871-thread-2) [ ] o.a.s.h.CdcrLeaderStateManager Received new leader state @ cdcr-cluster2:shard1
[junit4] 2> 258568 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35584/solr/cdcr-cluster2_shard2_replica_n2/ shard2
[junit4] 2> 258569 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35584/solr/cdcr-cluster2_shard1_replica_n1/ shard1
[junit4] 2> 258671 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 258673 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node3&name=cdcr-cluster2_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1245
[junit4] 2> 258720 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 258722 INFO (qtp1299351791-2802) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=cdcr-cluster2&newCollection=true&collection=cdcr-cluster2&version=2&replicaType=NRT&coreNodeName=core_node4&name=cdcr-cluster2_shard2_replica_n2&action=CREATE&numShards=2&shard=shard2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin} status=0 QTime=1284
[junit4] 2> 258725 INFO (qtp1299351791-2807) [n:127.0.0.1:35584_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 258822 INFO (zkCallback-871-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster2/state.json] for collection [cdcr-cluster2] has occurred - updating... (live nodes size: [1])
[junit4] 2> 259110 INFO (OverseerCollectionConfigSetProcessor-72307226829914115-127.0.0.1:35584_solr-n_0000000000) [n:127.0.0.1:35584_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 259725 INFO (qtp1299351791-2807) [n:127.0.0.1:35584_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={collection.configName=cdcr-cluster2&maxShardsPerNode=2&name=cdcr-cluster2&nrtReplicas=1&action=CREATE&numShards=2&property.solr.directoryFactory=solr.StandardDirectoryFactory&wt=javabin&version=2} status=0 QTime=2620
[junit4] 2> 259733 INFO (zkConnectionManagerCallback-931-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 259735 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 259736 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39460/solr ready
[junit4] 2> 259769 INFO (qtp1299351791-2804) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 259769 INFO (qtp1299351791-2804) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 259776 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 259776 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 259777 INFO (qtp1299351791-2806) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={_stateVer_=cdcr-cluster2:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=39
[junit4] 2> 259777 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-cluster2 with checkpoint -1 @ cdcr-cluster1:shard2
[junit4] 2> 259777 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2, shard: shard2
[junit4] 2> 259780 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Submitting bootstrap task to executor
[junit4] 2> 259783 INFO (zkCallback-896-thread-1) [ ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster1/cdcr/state/process @ cdcr-cluster1:shard1
[junit4] 2> 259784 INFO (zkCallback-896-thread-2) [ ] o.a.s.h.CdcrProcessStateManager The CDCR process state has changed: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/cdcr-cluster1/cdcr/state/process @ cdcr-cluster1:shard2
[junit4] 2> 259784 INFO (zkCallback-896-thread-1) [ ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-cluster1:shard1
[junit4] 2> 259784 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/cdcr params={qt=/cdcr&_stateVer_=cdcr-cluster1:4&action=start&wt=javabin&version=2} status=0 QTime=54
[junit4] 2> 259784 INFO (zkCallback-896-thread-2) [ ] o.a.s.h.CdcrProcessStateManager Received new CDCR process state from watcher: STARTED @ cdcr-cluster1:shard2
[junit4] 2> 259784 INFO (cdcr-bootstrap-status-927-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2 shard: shard2 leader: http://127.0.0.1:35584/solr/cdcr-cluster2_shard2_replica_n2/
[junit4] 2> 259788 INFO (qtp1299351791-2804) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={qt=/cdcr&masterUrl=http://127.0.0.1:45524/solr/cdcr-cluster1_shard2_replica_n2/&action=BOOTSTRAP&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 259790 INFO (zkConnectionManagerCallback-936-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 259791 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 259793 INFO (zkCallback-896-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 259793 INFO (zkCallback-896-thread-1) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:39460/solr ready
[junit4] 2> 259794 INFO (qtp1299351791-2806) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 259802 INFO (cdcr-bootstrap-status-927-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager CDCR bootstrap running for 1 seconds, sleeping for 2000 ms
[junit4] 2> 259806 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1611158354476924928,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 259807 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 259807 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 259808 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 259810 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 259810 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 259810 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 259810 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 259810 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 259810 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 259816 INFO (qtp1299351791-2806) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 259816 INFO (qtp1299351791-2806) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 259830 INFO (qtp1299351791-2804) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 259830 INFO (qtp1299351791-2804) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 259841 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={_stateVer_=cdcr-cluster2:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=39
[junit4] 2> 259842 INFO (zkCallback-896-thread-1) [ ] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-cluster2 with checkpoint -1 @ cdcr-cluster1:shard1
[junit4] 2> 259846 INFO (zkCallback-896-thread-1) [ ] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2, shard: shard1
[junit4] 2> 259846 INFO (zkCallback-896-thread-1) [ ] o.a.s.h.CdcrReplicatorManager Submitting bootstrap task to executor
[junit4] 2> 259842 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@20876bce[cdcr-cluster2_shard2_replica_n2] main]
[junit4] 2> 259847 INFO (searcherExecutor-1453-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.SolrCore [cdcr-cluster2_shard2_replica_n2] Registered new searcher Searcher@20876bce[cdcr-cluster2_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 259865 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard2_replica_n2 c:cdcr-cluster2 s:shard2 r:core_node4) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler No replay needed.
[junit4] 2> 259865 INFO (cdcr-bootstrap-status-932-thread-1) [ ] o.a.s.h.CdcrReplicatorManager Attempting to bootstrap target collection: cdcr-cluster2 shard: shard1 leader: http://127.0.0.1:35584/solr/cdcr-cluster2_shard1_replica_n1/
[junit4] 2> 259887 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.u.UpdateLog Starting to buffer updates. FSUpdateLog{state=ACTIVE, tlog=null}
[junit4] 2> 259887 INFO (qtp1299351791-2806) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&masterUrl=http://127.0.0.1:45524/solr/cdcr-cluster1_shard1_replica_n1/&action=BOOTSTRAP&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 259889 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 259890 INFO (cdcr-bootstrap-status-932-thread-1) [ ] o.a.s.h.CdcrReplicatorManager CDCR bootstrap running for 1 seconds, sleeping for 2000 ms
[junit4] 2> 259904 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1611158354579685376,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 259905 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 No uncommitted changes. Skipping IW.commit.
[junit4] 2> 259905 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 259905 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/update params={waitSearcher=true&openSearcher=false&commit=true&softCommit=false&commit_end_point=true&wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 259907 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
[junit4] 2> 259907 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Master's generation: 1
[junit4] 2> 259907 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Master's version: 0
[junit4] 2> 259907 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's generation: 1
[junit4] 2> 259908 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher Slave's version: 0
[junit4] 2> 259908 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.IndexFetcher New index in Master. Deleting mine...
[junit4] 2> 259915 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4974943c[cdcr-cluster2_shard1_replica_n1] main]
[junit4] 2> 259917 INFO (searcherExecutor-1454-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-cluster2_shard1_replica_n1] Registered new searcher Searcher@4974943c[cdcr-cluster2_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 259917 INFO (recoveryExecutor-868-thread-1-processing-n:127.0.0.1:35584_solr x:cdcr-cluster2_shard1_replica_n1 c:cdcr-cluster2 s:shard1 r:core_node3) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrRequestHandler No replay needed.
[junit4] 2> 261785 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.c.CdcrBidirectionalTest Adding 10 docs with commit=true, numDocs=100
[junit4] 2> 261803 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 261804 INFO (cdcr-bootstrap-status-927-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager CDCR bootstrap successful in 3 seconds
[junit4] 2> 261825 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster1/terms/shard1 to Terms{values={core_node3=1}, version=1}
[junit4] 2> 261825 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-cluster1:4&wt=javabin&version=2} status=0 QTime=36
[junit4] 2> 261829 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.ZkShardTerms Successful update of terms at /collections/cdcr-cluster1/terms/shard2 to Terms{values={core_node4=1}, version=1}
[junit4] 2> 261829 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/update params={_stateVer_=cdcr-cluster1:4&wt=javabin&version=2} status=0 QTime=41
[junit4] 2> 261832 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 261832 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 261835 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 261835 INFO (qtp1299351791-2801) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 261836 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1611158356605534208,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 261836 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@73873668 commitCommandVersion:1611158356605534208
[junit4] 2> 261838 INFO (qtp1299351791-2806) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={_stateVer_=cdcr-cluster2:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=28
[junit4] 2> 261839 INFO (cdcr-bootstrap-status-927-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-cluster2 with checkpoint -1 @ cdcr-cluster1:shard2
[junit4] 2> 261839 INFO (cdcr-bootstrap-status-927-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.h.CdcrReplicatorManager Bootstrap successful, giving the go-ahead to replicator
[junit4] 2> 261840 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1611158356609728512,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 261840 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@29ac76a6 commitCommandVersion:1611158356609728512
[junit4] 2> 261879 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5b12b0db[cdcr-cluster1_shard1_replica_n1] main]
[junit4] 2> 261881 INFO (searcherExecutor-1441-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard1_replica_n1 c:cdcr-cluster1 s:shard1 r:core_node3) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-cluster1_shard1_replica_n1] Registered new searcher Searcher@5b12b0db[cdcr-cluster1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C43)))}
[junit4] 2> 261881 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 261881 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&update.chain=cdcr-processor-chain&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:45524/solr/cdcr-cluster1_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false} status=0 QTime=41
[junit4] 2> 261890 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@23ad4d7e[cdcr-cluster1_shard2_replica_n2] main]
[junit4] 2> 261891 INFO (searcherExecutor-1442-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard2_replica_n2 c:cdcr-cluster1 s:shard2 r:core_node4) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.SolrCore [cdcr-cluster1_shard2_replica_n2] Registered new searcher Searcher@23ad4d7e[cdcr-cluster1_shard2_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C57)))}
[junit4] 2> 261891 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 261891 INFO (qtp1391551120-2859) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/update params={update.distrib=FROMLEADER&update.chain=cdcr-processor-chain&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:45524/solr/cdcr-cluster1_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false} status=0 QTime=55
[junit4] 2> 261892 INFO (qtp1391551120-2858) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/update params={_stateVer_=cdcr-cluster1:4&waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2} status=0 QTime=62
[junit4] 2> 261893 INFO (TEST-CdcrBidirectionalTest.testBiDir-seed#[AE806A5A97EFF027]) [ ] o.a.s.c.c.CdcrBidirectionalTest Adding 10 docs with commit=true, numDocs=200
[junit4] 2> 261899 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/update params={_stateVer_=cdcr-cluster1:4&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 261900 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={qt=/cdcr&action=BOOTSTRAP_STATUS&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 261901 INFO (cdcr-bootstrap-status-932-thread-1) [ ] o.a.s.h.CdcrReplicatorManager CDCR bootstrap successful in 3 seconds
[junit4] 2> 261905 INFO (qtp1391551120-2856) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster1_shard2_replica_n2] webapp=/solr path=/update params={_stateVer_=cdcr-cluster1:4&wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 261917 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1611158356690468864,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 261919 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1611158356692566016,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 261920 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@29ac76a6 commitCommandVersion:1611158356690468864
[junit4] 2> 261921 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@73873668 commitCommandVersion:1611158356692566016
[junit4] 2> 261936 INFO (qtp1299351791-2804) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 261936 INFO (qtp1299351791-2804) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard1 r:core_node3 x:cdcr-cluster2_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster2_shard1_replica_n1] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 261937 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.h.CdcrRequestHandler Found maxVersionFromRecent 0 maxVersionFromIndex 0
[junit4] 2> 261937 INFO (qtp1299351791-2808) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={action=SHARDCHECKPOINT&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 261938 INFO (qtp1299351791-2806) [n:127.0.0.1:35584_solr c:cdcr-cluster2 s:shard2 r:core_node4 x:cdcr-cluster2_shard2_replica_n2] o.a.s.c.S.Request [cdcr-cluster2_shard2_replica_n2] webapp=/solr path=/cdcr params={_stateVer_=cdcr-cluster2:4&action=COLLECTIONCHECKPOINT&wt=javabin&version=2} status=0 QTime=20
[junit4] 2> 261938 INFO (cdcr-bootstrap-status-932-thread-1) [ ] o.a.s.h.CdcrReplicatorManager Create new update log reader for target cdcr-cluster2 with checkpoint -1 @ cdcr-cluster1:shard1
[junit4] 2> 261939 INFO (cdcr-bootstrap-status-932-thread-1) [ ] o.a.s.h.CdcrReplicatorManager Bootstrap successful, giving the go-ahead to replicator
[junit4] 2> 261964 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@12a8ed31[cdcr-cluster1_shard1_replica_n1] main]
[junit4] 2> 261965 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 261966 INFO (searcherExecutor-1441-thread-1-processing-n:127.0.0.1:45524_solr x:cdcr-cluster1_shard1_replica_n1 c:cdcr-cluster1 s:shard1 r:core_node3) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.SolrCore [cdcr-cluster1_shard1_replica_n1] Registered new searcher Searcher@12a8ed31[cdcr-cluster1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(8.0.0):C43) Uninverting(_1(8.0.0):C54)))}
[junit4] 2> 261966 INFO (qtp1391551120-2862) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard1 r:core_node3 x:cdcr-cluster1_shard1_replica_n1] o.a.s.c.S.Request [cdcr-cluster1_shard1_replica_n1] webapp=/solr path=/update params={update.distrib=FROMLEADER&update.chain=cdcr-processor-chain&waitSearcher=true&openSearcher=true&commit=true&softCommit=false&distrib.from=http://127.0.0.1:45524/solr/cdcr-cluster1_shard2_replica_n2/&commit_end_point=true&wt=javabin&version=2&expungeDeletes=false} status=0 QTime=59
[junit4] 2> 261967 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7edd014b[cdcr-cluster1_shard2_replica_n2] main]
[junit4] 2> 261967 INFO (qtp1391551120-2861) [n:127.0.0.1:45524_solr c:cdcr-cluster1 s:shard2 r:core_node4 x:cdcr-cluster1_shard2_replica_n2] o.a.s.u.Direct
[...truncated too long message...]
ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1895415044
[copy] Copying 238 files to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1895415044
[delete] Deleting directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1895415044
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 20ms :: artifacts dl 1ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 92 minutes 32 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)
[JENKINS] Lucene-Solr-Tests-master - Build # 2794 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-master/2794/
3 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimLargeCluster.testNodeLost
Error Message:
last state: DocCollection(testNodeLost//clusterstate.json/453)={ "replicationFactor":"1", "pullReplicas":"0", "router":{"name":"compositeId"}, "maxShardsPerNode":"5", "autoAddReplicas":"false", "nrtReplicas":"10", "tlogReplicas":"0", "autoCreated":"true", "shards":{ "shard20":{ "replicas":{ "core_node191":{ "core":"testNodeLost_shard20_replica_n191", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10009_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node206":{ "node_name":"127.0.0.1:10059_solr", "core":"testNodeLost_shard20_replica_n205", "state":"active", "INDEX.sizeInBytes":10240, "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6}, "core_node199":{ "core":"testNodeLost_shard20_replica_n199", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10017_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node198":{ "core":"testNodeLost_shard20_replica_n198", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10026_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node197":{ "core":"testNodeLost_shard20_replica_n197", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10047_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node196":{ "core":"testNodeLost_shard20_replica_n196", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10091_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node195":{ "core":"testNodeLost_shard20_replica_n195", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10058_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node194":{ "core":"testNodeLost_shard20_replica_n194", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10041_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node193":{ "core":"testNodeLost_shard20_replica_n193", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10043_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node192":{ "core":"testNodeLost_shard20_replica_n192", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10054_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"73330000-7fffffff", "state":"active"}, "shard10":{ "replicas":{ "core_node92":{ "core":"testNodeLost_shard10_replica_n92", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10054_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node91":{ "core":"testNodeLost_shard10_replica_n91", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10009_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node94":{ "core":"testNodeLost_shard10_replica_n94", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10041_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node93":{ "core":"testNodeLost_shard10_replica_n93", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10043_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node96":{ "core":"testNodeLost_shard10_replica_n96", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10091_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node95":{ "core":"testNodeLost_shard10_replica_n95", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10058_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node98":{ "core":"testNodeLost_shard10_replica_n98", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10026_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node97":{ "core":"testNodeLost_shard10_replica_n97", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10047_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node99":{ "core":"testNodeLost_shard10_replica_n99", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10017_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node208":{ "node_name":"127.0.0.1:10053_solr", "core":"testNodeLost_shard10_replica_n207", "state":"active", "INDEX.sizeInBytes":10240, "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6}}, "range":"f3330000-ffffffff", "state":"active"}, "shard11":{ "replicas":{ "core_node108":{ "core":"testNodeLost_shard11_replica_n108", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10060_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node107":{ "core":"testNodeLost_shard11_replica_n107", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10001_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node106":{ "core":"testNodeLost_shard11_replica_n106", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10023_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node105":{ "core":"testNodeLost_shard11_replica_n105", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10085_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node104":{ "core":"testNodeLost_shard11_replica_n104", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10061_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node103":{ "core":"testNodeLost_shard11_replica_n103", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10024_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node102":{ "core":"testNodeLost_shard11_replica_n102", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10072_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node101":{ "core":"testNodeLost_shard11_replica_n101", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10035_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node110":{ "core":"testNodeLost_shard11_replica_n110", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10028_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node109":{ "core":"testNodeLost_shard11_replica_n109", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10040_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"0-ccbffff", "state":"active"}, "shard12":{ "replicas":{ "core_node119":{ "core":"testNodeLost_shard12_replica_n119", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10000_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node118":{ "core":"testNodeLost_shard12_replica_n118", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10045_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node117":{ "core":"testNodeLost_shard12_replica_n117", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10084_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node116":{ "core":"testNodeLost_shard12_replica_n116", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10046_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node115":{ "core":"testNodeLost_shard12_replica_n115", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10056_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node114":{ "core":"testNodeLost_shard12_replica_n114", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10070_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node113":{ "core":"testNodeLost_shard12_replica_n113", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10077_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node112":{ "core":"testNodeLost_shard12_replica_n112", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10092_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node111":{ "core":"testNodeLost_shard12_replica_n111", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10031_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node120":{ "core":"testNodeLost_shard12_replica_n120", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10016_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"ccc0000-1998ffff", "state":"active"}, "shard2":{ "replicas":{ "core_node18":{ "core":"testNodeLost_shard2_replica_n18", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10045_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node17":{ "core":"testNodeLost_shard2_replica_n17", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10084_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node19":{ "core":"testNodeLost_shard2_replica_n19", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10000_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node20":{ "core":"testNodeLost_shard2_replica_n20", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10016_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node12":{ "core":"testNodeLost_shard2_replica_n12", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10092_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node11":{ "core":"testNodeLost_shard2_replica_n11", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10031_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node14":{ "core":"testNodeLost_shard2_replica_n14", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10070_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node13":{ "core":"testNodeLost_shard2_replica_n13", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10077_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node16":{ "core":"testNodeLost_shard2_replica_n16", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10046_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node15":{ "core":"testNodeLost_shard2_replica_n15", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10056_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"8ccc0000-9998ffff", "state":"active"}, "shard13":{ "replicas":{ "core_node129":{ "core":"testNodeLost_shard13_replica_n129", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10015_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node128":{ "core":"testNodeLost_shard13_replica_n128", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10011_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node127":{ "core":"testNodeLost_shard13_replica_n127", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10089_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node126":{ "core":"testNodeLost_shard13_replica_n126", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10032_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node125":{ "core":"testNodeLost_shard13_replica_n125", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10075_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node124":{ "core":"testNodeLost_shard13_replica_n124", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10051_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node123":{ "core":"testNodeLost_shard13_replica_n123", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10039_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node122":{ "core":"testNodeLost_shard13_replica_n122", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10044_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node121":{ "core":"testNodeLost_shard13_replica_n121", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10002_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node130":{ "core":"testNodeLost_shard13_replica_n130", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10012_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"19990000-2665ffff", "state":"active"}, "shard3":{ "replicas":{ "core_node29":{ "core":"testNodeLost_shard3_replica_n29", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10015_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node28":{ "core":"testNodeLost_shard3_replica_n28", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10011_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node30":{ "core":"testNodeLost_shard3_replica_n30", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10012_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node21":{ "core":"testNodeLost_shard3_replica_n21", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10002_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node23":{ "core":"testNodeLost_shard3_replica_n23", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10039_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node22":{ "core":"testNodeLost_shard3_replica_n22", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10044_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node25":{ "core":"testNodeLost_shard3_replica_n25", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10075_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node24":{ "core":"testNodeLost_shard3_replica_n24", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10051_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node27":{ "core":"testNodeLost_shard3_replica_n27", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10089_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node26":{ "core":"testNodeLost_shard3_replica_n26", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10032_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"99990000-a665ffff", "state":"active"}, "shard14":{ "replicas":{ "core_node139":{ "core":"testNodeLost_shard14_replica_n139", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10088_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node138":{ "core":"testNodeLost_shard14_replica_n138", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10093_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node137":{ "core":"testNodeLost_shard14_replica_n137", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10067_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node136":{ "core":"testNodeLost_shard14_replica_n136", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10014_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node135":{ "core":"testNodeLost_shard14_replica_n135", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10094_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node134":{ "core":"testNodeLost_shard14_replica_n134", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10007_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node133":{ "core":"testNodeLost_shard14_replica_n133", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10018_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node132":{ "core":"testNodeLost_shard14_replica_n132", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10038_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node131":{ "core":"testNodeLost_shard14_replica_n131", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10048_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node140":{ "core":"testNodeLost_shard14_replica_n140", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10019_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"26660000-3332ffff", "state":"active"}, "shard15":{ "replicas":{ "core_node149":{ "core":"testNodeLost_shard15_replica_n149", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10036_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node148":{ "core":"testNodeLost_shard15_replica_n148", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10029_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node147":{ "core":"testNodeLost_shard15_replica_n147", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10071_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node146":{ "core":"testNodeLost_shard15_replica_n146", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10065_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node145":{ "core":"testNodeLost_shard15_replica_n145", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10097_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node144":{ "core":"testNodeLost_shard15_replica_n144", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10079_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node143":{ "core":"testNodeLost_shard15_replica_n143", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10096_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node142":{ "core":"testNodeLost_shard15_replica_n142", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10095_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node141":{ "core":"testNodeLost_shard15_replica_n141", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10010_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node150":{ "core":"testNodeLost_shard15_replica_n150", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10062_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"33330000-3fffffff", "state":"active"}, "shard4":{ "replicas":{ "core_node39":{ "core":"testNodeLost_shard4_replica_n39", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10088_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node40":{ "core":"testNodeLost_shard4_replica_n40", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10019_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node32":{ "core":"testNodeLost_shard4_replica_n32", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10038_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node31":{ "core":"testNodeLost_shard4_replica_n31", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10048_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node34":{ "core":"testNodeLost_shard4_replica_n34", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10007_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node33":{ "core":"testNodeLost_shard4_replica_n33", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10018_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node36":{ "core":"testNodeLost_shard4_replica_n36", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10014_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node35":{ "core":"testNodeLost_shard4_replica_n35", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10094_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node38":{ "core":"testNodeLost_shard4_replica_n38", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10093_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node37":{ "core":"testNodeLost_shard4_replica_n37", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10067_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"a6660000-b332ffff", "state":"active"}, "shard5":{ "replicas":{ "core_node50":{ "core":"testNodeLost_shard5_replica_n50", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10062_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node41":{ "core":"testNodeLost_shard5_replica_n41", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10010_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node43":{ "core":"testNodeLost_shard5_replica_n43", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10096_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node42":{ "core":"testNodeLost_shard5_replica_n42", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10095_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node45":{ "core":"testNodeLost_shard5_replica_n45", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10097_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node44":{ "core":"testNodeLost_shard5_replica_n44", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10079_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node47":{ "core":"testNodeLost_shard5_replica_n47", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10071_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node46":{ "core":"testNodeLost_shard5_replica_n46", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10065_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node49":{ "core":"testNodeLost_shard5_replica_n49", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10036_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node48":{ "core":"testNodeLost_shard5_replica_n48", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10029_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"b3330000-bfffffff", "state":"active"}, "shard16":{ "replicas":{ "core_node159":{ "core":"testNodeLost_shard16_replica_n159", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10005_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node158":{ "core":"testNodeLost_shard16_replica_n158", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10074_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node157":{ "core":"testNodeLost_shard16_replica_n157", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10076_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node156":{ "core":"testNodeLost_shard16_replica_n156", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10099_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node155":{ "core":"testNodeLost_shard16_replica_n155", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10078_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node154":{ "core":"testNodeLost_shard16_replica_n154", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10081_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node153":{ "core":"testNodeLost_shard16_replica_n153", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10022_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node152":{ "core":"testNodeLost_shard16_replica_n152", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10049_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node151":{ "core":"testNodeLost_shard16_replica_n151", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10008_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node160":{ "core":"testNodeLost_shard16_replica_n160", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10057_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"40000000-4ccbffff", "state":"active"}, "shard17":{ "replicas":{ "core_node169":{ "core":"testNodeLost_shard17_replica_n169", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10066_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node168":{ "core":"testNodeLost_shard17_replica_n168", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10086_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node167":{ "core":"testNodeLost_shard17_replica_n167", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10052_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node166":{ "core":"testNodeLost_shard17_replica_n166", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10050_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node165":{ "core":"testNodeLost_shard17_replica_n165", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10064_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node164":{ "core":"testNodeLost_shard17_replica_n164", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10098_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node163":{ "core":"testNodeLost_shard17_replica_n163", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10087_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node162":{ "core":"testNodeLost_shard17_replica_n162", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10069_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node161":{ "core":"testNodeLost_shard17_replica_n161", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10013_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node170":{ "core":"testNodeLost_shard17_replica_n170", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10006_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"4ccc0000-5998ffff", "state":"active"}, "shard18":{ "replicas":{ "core_node180":{ "core":"testNodeLost_shard18_replica_n180", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10003_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node179":{ "core":"testNodeLost_shard18_replica_n179", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10020_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node178":{ "core":"testNodeLost_shard18_replica_n178", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10073_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node177":{ "core":"testNodeLost_shard18_replica_n177", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10083_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node176":{ "core":"testNodeLost_shard18_replica_n176", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10080_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node175":{ "core":"testNodeLost_shard18_replica_n175", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10021_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node174":{ "core":"testNodeLost_shard18_replica_n174", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10033_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node173":{ "core":"testNodeLost_shard18_replica_n173", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10053_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node172":{ "core":"testNodeLost_shard18_replica_n172", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10030_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node171":{ "core":"testNodeLost_shard18_replica_n171", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10037_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"59990000-6665ffff", "state":"active"}, "shard19":{ "replicas":{ "core_node190":{ "core":"testNodeLost_shard19_replica_n190", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10068_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node202":{ "node_name":"127.0.0.1:10072_solr", "core":"testNodeLost_shard19_replica_n201", "state":"active", "INDEX.sizeInBytes":10240, "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6}, "core_node189":{ "core":"testNodeLost_shard19_replica_n189", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10059_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node188":{ "core":"testNodeLost_shard19_replica_n188", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10090_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node187":{ "core":"testNodeLost_shard19_replica_n187", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10034_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node185":{ "core":"testNodeLost_shard19_replica_n185", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10042_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node184":{ "core":"testNodeLost_shard19_replica_n184", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10027_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node183":{ "core":"testNodeLost_shard19_replica_n183", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10055_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node182":{ "core":"testNodeLost_shard19_replica_n182", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10025_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node181":{ "core":"testNodeLost_shard19_replica_n181", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10063_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"66660000-7332ffff", "state":"active"}, "shard1":{ "replicas":{ "core_node1":{ "core":"testNodeLost_shard1_replica_n1", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10035_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node6":{ "core":"testNodeLost_shard1_replica_n6", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10023_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node10":{ "core":"testNodeLost_shard1_replica_n10", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10028_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node7":{ "core":"testNodeLost_shard1_replica_n7", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10001_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node8":{ "core":"testNodeLost_shard1_replica_n8", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10060_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node9":{ "core":"testNodeLost_shard1_replica_n9", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10040_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node2":{ "core":"testNodeLost_shard1_replica_n2", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10072_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node3":{ "core":"testNodeLost_shard1_replica_n3", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10024_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node4":{ "core":"testNodeLost_shard1_replica_n4", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10061_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node5":{ "core":"testNodeLost_shard1_replica_n5", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10085_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"80000000-8ccbffff", "state":"active"}, "shard6":{ "replicas":{ "core_node60":{ "core":"testNodeLost_shard6_replica_n60", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10057_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node52":{ "core":"testNodeLost_shard6_replica_n52", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10049_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node51":{ "core":"testNodeLost_shard6_replica_n51", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10008_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node54":{ "core":"testNodeLost_shard6_replica_n54", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10081_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node53":{ "core":"testNodeLost_shard6_replica_n53", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10022_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node56":{ "core":"testNodeLost_shard6_replica_n56", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10099_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node55":{ "core":"testNodeLost_shard6_replica_n55", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10078_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node58":{ "core":"testNodeLost_shard6_replica_n58", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10074_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node57":{ "core":"testNodeLost_shard6_replica_n57", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10076_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node59":{ "core":"testNodeLost_shard6_replica_n59", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10005_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"c0000000-cccbffff", "state":"active"}, "shard7":{ "replicas":{ "core_node70":{ "core":"testNodeLost_shard7_replica_n70", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10006_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node61":{ "core":"testNodeLost_shard7_replica_n61", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10013_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node63":{ "core":"testNodeLost_shard7_replica_n63", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10087_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node62":{ "core":"testNodeLost_shard7_replica_n62", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10069_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node65":{ "core":"testNodeLost_shard7_replica_n65", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10064_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node64":{ "core":"testNodeLost_shard7_replica_n64", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10098_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node67":{ "core":"testNodeLost_shard7_replica_n67", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10052_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node66":{ "core":"testNodeLost_shard7_replica_n66", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10050_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node69":{ "core":"testNodeLost_shard7_replica_n69", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10066_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node68":{ "core":"testNodeLost_shard7_replica_n68", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10086_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"cccc0000-d998ffff", "state":"active"}, "shard8":{ "replicas":{ "core_node80":{ "core":"testNodeLost_shard8_replica_n80", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10003_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node72":{ "core":"testNodeLost_shard8_replica_n72", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10030_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node71":{ "core":"testNodeLost_shard8_replica_n71", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10037_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node74":{ "core":"testNodeLost_shard8_replica_n74", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10033_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node73":{ "core":"testNodeLost_shard8_replica_n73", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10053_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node76":{ "core":"testNodeLost_shard8_replica_n76", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10080_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node75":{ "core":"testNodeLost_shard8_replica_n75", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10021_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node78":{ "core":"testNodeLost_shard8_replica_n78", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10073_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node77":{ "core":"testNodeLost_shard8_replica_n77", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10083_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node79":{ "core":"testNodeLost_shard8_replica_n79", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10020_solr", "state":"down", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"d9990000-e665ffff", "state":"active"}, "shard9":{ "replicas":{ "core_node90":{ "core":"testNodeLost_shard9_replica_n90", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10068_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node81":{ "core":"testNodeLost_shard9_replica_n81", "leader":"true", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10063_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node204":{ "node_name":"127.0.0.1:10056_solr", "core":"testNodeLost_shard9_replica_n203", "state":"active", "INDEX.sizeInBytes":10240, "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6}, "core_node83":{ "core":"testNodeLost_shard9_replica_n83", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10055_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node82":{ "core":"testNodeLost_shard9_replica_n82", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10025_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node85":{ "core":"testNodeLost_shard9_replica_n85", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10042_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node84":{ "core":"testNodeLost_shard9_replica_n84", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10027_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node87":{ "core":"testNodeLost_shard9_replica_n87", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10034_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node89":{ "core":"testNodeLost_shard9_replica_n89", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10059_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}, "core_node88":{ "core":"testNodeLost_shard9_replica_n88", "SEARCHER.searcher.maxDoc":0, "SEARCHER.searcher.deletedDocs":0, "INDEX.sizeInBytes":10240, "node_name":"127.0.0.1:10090_solr", "state":"active", "type":"NRT", "INDEX.sizeInGB":9.5367431640625E-6, "SEARCHER.searcher.numDocs":0}}, "range":"e6660000-f332ffff", "state":"active"}}}
Stack Trace:
java.util.concurrent.TimeoutException: last state: DocCollection(testNodeLost//clusterstate.json/453)={
"replicationFactor":"1",
"pullReplicas":"0",
"router":{"name":"compositeId"},
"maxShardsPerNode":"5",
"autoAddReplicas":"false",
"nrtReplicas":"10",
"tlogReplicas":"0",
"autoCreated":"true",
"shards":{
"shard20":{
"replicas":{
"core_node191":{
"core":"testNodeLost_shard20_replica_n191",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10009_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node206":{
"node_name":"127.0.0.1:10059_solr",
"core":"testNodeLost_shard20_replica_n205",
"state":"active",
"INDEX.sizeInBytes":10240,
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6},
"core_node199":{
"core":"testNodeLost_shard20_replica_n199",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10017_solr",
"state":"down",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node198":{
"core":"testNodeLost_shard20_replica_n198",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10026_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node197":{
"core":"testNodeLost_shard20_replica_n197",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10047_solr",
"state":"down",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node196":{
"core":"testNodeLost_shard20_replica_n196",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10091_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node195":{
"core":"testNodeLost_shard20_replica_n195",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10058_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node194":{
"core":"testNodeLost_shard20_replica_n194",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10041_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node193":{
"core":"testNodeLost_shard20_replica_n193",
"leader":"true",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10043_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node192":{
"core":"testNodeLost_shard20_replica_n192",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10054_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0}},
"range":"73330000-7fffffff",
"state":"active"},
"shard10":{
"replicas":{
"core_node92":{
"core":"testNodeLost_shard10_replica_n92",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10054_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node91":{
"core":"testNodeLost_shard10_replica_n91",
"leader":"true",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10009_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node94":{
"core":"testNodeLost_shard10_replica_n94",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10041_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node93":{
"core":"testNodeLost_shard10_replica_n93",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10043_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node96":{
"core":"testNodeLost_shard10_replica_n96",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10091_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node95":{
"core":"testNodeLost_shard10_replica_n95",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10058_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node98":{
"core":"testNodeLost_shard10_replica_n98",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10026_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node97":{
"core":"testNodeLost_shard10_replica_n97",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10047_solr",
"state":"down",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node99":{
"core":"testNodeLost_shard10_replica_n99",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10017_solr",
"state":"down",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node208":{
"node_name":"127.0.0.1:10053_solr",
"core":"testNodeLost_shard10_replica_n207",
"state":"active",
"INDEX.sizeInBytes":10240,
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6}},
"range":"f3330000-ffffffff",
"state":"active"},
"shard11":{
"replicas":{
"core_node108":{
"core":"testNodeLost_shard11_replica_n108",
"leader":"true",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10060_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node107":{
"core":"testNodeLost_shard11_replica_n107",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10001_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node106":{
"core":"testNodeLost_shard11_replica_n106",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10023_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node105":{
"core":"testNodeLost_shard11_replica_n105",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10085_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node104":{
"core":"testNodeLost_shard11_replica_n104",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10061_solr",
"state":"active",
"type":"NRT",
"INDEX.sizeInGB":9.5367431640625E-6,
"SEARCHER.searcher.numDocs":0},
"core_node103":{
"core":"testNodeLost_shard11_replica_n103",
"SEARCHER.searcher.maxDoc":0,
"SEARCHER.searcher.deletedDocs":0,
"INDEX.sizeInBytes":10240,
"node_name":"127.0.0.1:10024_solr",
"state":"active",
"type":"NRT",
[...truncated too long message...]
ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
ivy-availability-check:
[loadresource] Do not set property disallowed.ivy.jars.list as its length is 0.
-ivy-fail-disallowed-ivy-version:
ivy-fail:
ivy-fail:
ivy-configure:
[ivy:configure] :: loading settings :: file = /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/lucene/top-level-ivy-settings.xml
resolve:
jar-checksums:
[mkdir] Created dir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1051895538
[copy] Copying 238 files to /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1051895538
[delete] Deleting directory /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-master/solr/null1051895538
check-working-copy:
[ivy:cachepath] :: resolving dependencies :: org.eclipse.jgit#org.eclipse.jgit-caller;working
[ivy:cachepath] confs: [default]
[ivy:cachepath] found org.eclipse.jgit#org.eclipse.jgit;4.6.0.201612231935-r in public
[ivy:cachepath] found com.jcraft#jsch;0.1.53 in public
[ivy:cachepath] found com.googlecode.javaewah#JavaEWAH;1.1.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpclient;4.3.6 in public
[ivy:cachepath] found org.apache.httpcomponents#httpcore;4.3.3 in public
[ivy:cachepath] found commons-logging#commons-logging;1.1.3 in public
[ivy:cachepath] found commons-codec#commons-codec;1.6 in public
[ivy:cachepath] found org.slf4j#slf4j-api;1.7.2 in public
[ivy:cachepath] :: resolution report :: resolve 87ms :: artifacts dl 1ms
---------------------------------------------------------------------
| | modules || artifacts |
| conf | number| search|dwnlded|evicted|| number|dwnlded|
---------------------------------------------------------------------
| default | 8 | 0 | 0 | 0 || 8 | 0 |
---------------------------------------------------------------------
[wc-checker] Initializing working copy...
[wc-checker] SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
[wc-checker] SLF4J: Defaulting to no-operation (NOP) logger implementation
[wc-checker] SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
[wc-checker] Checking working copy status...
-jenkins-base:
BUILD SUCCESSFUL
Total time: 217 minutes 26 seconds
Archiving artifacts
java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath$34.hasMatch(FilePath.java:2678)
at hudson.FilePath$34.invoke(FilePath.java:2557)
at hudson.FilePath$34.invoke(FilePath.java:2547)
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2918)
Also: hudson.remoting.Channel$CallSiteStackTrace: Remote call to lucene2
at hudson.remoting.Channel.attachCallSiteStackTrace(Channel.java:1741)
at hudson.remoting.UserRequest$ExceptionResponse.retrieve(UserRequest.java:357)
at hudson.remoting.Channel.call(Channel.java:955)
at hudson.FilePath.act(FilePath.java:1036)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
Caused: hudson.FilePath$TunneledInterruptedException
at hudson.FilePath$FileCallableWrapper.call(FilePath.java:2920)
at hudson.remoting.UserRequest.perform(UserRequest.java:212)
at hudson.remoting.UserRequest.perform(UserRequest.java:54)
at hudson.remoting.Request$2.run(Request.java:369)
at hudson.remoting.InterceptingExecutorService$1.call(InterceptingExecutorService.java:72)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:748)
Caused: java.lang.InterruptedException: java.lang.InterruptedException: no matches found within 10000
at hudson.FilePath.act(FilePath.java:1038)
at hudson.FilePath.act(FilePath.java:1025)
at hudson.FilePath.validateAntFileMask(FilePath.java:2547)
at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:243)
at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:81)
at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:744)
at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:690)
at hudson.model.Build$BuildExecution.post2(Build.java:186)
at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:635)
at hudson.model.Run.execute(Run.java:1819)
at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
at hudson.model.ResourceController.execute(ResourceController.java:97)
at hudson.model.Executor.run(Executor.java:429)
No artifacts found that match the file pattern "**/*.events,heapdumps/**,**/hs_err_pid*". Configuration error?
Recording test results
Build step 'Publish JUnit test result report' changed build result to UNSTABLE
Email was triggered for: Unstable (Test Failures)
Sending email for trigger: Unstable (Test Failures)