You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2019/02/20 21:35:22 UTC
[JENKINS-EA] Lucene-Solr-8.0-Linux
(64bit/jdk-12-ea+shipilev-fastdebug) - Build # 199 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-8.0-Linux/199/
Java: 64bit/jdk-12-ea+shipilev-fastdebug -XX:-UseCompressedOops -XX:+UseParallelGC
2 tests failed.
FAILED: org.apache.solr.cloud.TestCloudSearcherWarming.testRepFactor1LeaderStartup
Error Message:
No live SolrServers available to handle this request
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:343)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:213)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1110)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:517)
at org.apache.solr.cloud.TestCloudSearcherWarming.tearDown(TestCloudSearcherWarming.java:78)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:996)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:835)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testSearchRate
Error Message:
The trigger did not start in time
Stack Trace:
java.lang.AssertionError: The trigger did not start in time
at __randomizedtesting.SeedInfo.seed([D769C3EE39CB2E8F:8A21DD67F60D88C0]:0)
at org.junit.Assert.fail(Assert.java:88)
at org.junit.Assert.assertTrue(Assert.java:41)
at org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration.testSearchRate(TestSimTriggerIntegration.java:1369)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:567)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1750)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:938)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:974)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:988)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:947)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:832)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:883)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:894)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.base/java.lang.Thread.run(Thread.java:835)
Build Log:
[...truncated 13022 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[junit4] 2> Creating dataDir: /home/jenkins/workspace/Lucene-Solr-8.0-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestSimTriggerIntegration_D769C3EE39CB2E8F-001/init-core-data-001
[junit4] 2> 562322 WARN (SUITE-TestSimTriggerIntegration-seed#[D769C3EE39CB2E8F]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 562322 INFO (SUITE-TestSimTriggerIntegration-seed#[D769C3EE39CB2E8F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 562323 INFO (SUITE-TestSimTriggerIntegration-seed#[D769C3EE39CB2E8F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason="", ssl=0.0/0.0, value=0.0/0.0, clientAuth=0.0/0.0)
[junit4] 2> 562323 INFO (SUITE-TestSimTriggerIntegration-seed#[D769C3EE39CB2E8F]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 562327 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
[junit4] 2> 562348 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10200_solr
[junit4] 2> 562348 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 562348 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 562349 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 562349 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 562349 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10200_solr]
[junit4] 2> 562349 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 562349 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 562349 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 562350 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 562350 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 562350 DEBUG (ScheduledTrigger-1024-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 1
[junit4] 2> 562376 DEBUG (ScheduledTrigger-1024-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562396 DEBUG (ScheduledTrigger-1024-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562416 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562436 DEBUG (ScheduledTrigger-1024-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562456 DEBUG (ScheduledTrigger-1024-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562472 DEBUG (simCloudManagerPool-1023-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 562472 DEBUG (simCloudManagerPool-1023-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 562473 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 562474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 562474 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 562477 DEBUG (ScheduledTrigger-1024-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562497 DEBUG (ScheduledTrigger-1024-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562497 DEBUG (simCloudManagerPool-1023-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 562498 DEBUG (simCloudManagerPool-1023-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 562498 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 562498 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 562498 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10200_solr, 127.0.0.1:10201_solr]
[junit4] 2> 562498 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=500000, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 562498 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 562499 DEBUG (ScheduledTrigger-1024-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 562499 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 562499 DEBUG (ScheduledTrigger-1024-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 562503 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10202_solr
[junit4] 2> 562517 DEBUG (ScheduledTrigger-1024-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 562528 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 562528 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562528 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10202_solr at time 268735778259963
[junit4] 2> 562537 DEBUG (ScheduledTrigger-1024-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562549 DEBUG (ScheduledTrigger-1024-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 562549 DEBUG (ScheduledTrigger-1024-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562553 DEBUG (simCloudManagerPool-1023-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 562553 DEBUG (simCloudManagerPool-1023-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 562553 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 562554 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 562554 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger - Initial livenodes: [127.0.0.1:10200_solr, 127.0.0.1:10201_solr, 127.0.0.1:10202_solr]
[junit4] 2> 562554 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=2, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 562554 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 562554 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 562554 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562554 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 562558 DEBUG (ScheduledTrigger-1024-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562574 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 562574 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562574 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:10202_solr] added at times [268735778259963], now=268738076438063
[junit4] 2> 562578 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f469ec9a03fbT7r8fx4arr81op6sa0eeakebla",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":268735778259963,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268735778259963],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268738085831263,
[junit4] 2> "nodeNames":["127.0.0.1:10202_solr"]}}
[junit4] 2> 562578 DEBUG (ScheduledTrigger-1024-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10202_solr
[junit4] 2> 562579 DEBUG (AutoscalingActionExecutor-1025-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f469ec9a03fbT7r8fx4arr81op6sa0eeakebla",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":268735778259963,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268735778259963],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268738085831263,
[junit4] 2> "nodeNames":["127.0.0.1:10202_solr"]}}
[junit4] 2> 562580 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2056) ~[?:?]
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2090) ~[?:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:201) [java/:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:835) [?:?]
[junit4] 2> 562584 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 562585 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
[junit4] 2> 562590 INFO (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventFromRestoredState
[junit4] 2> 562593 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10203_solr
[junit4] 2> 562594 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 562594 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 562599 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 562600 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 562600 INFO (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 562617 DEBUG (simCloudManagerPool-1029-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 562662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 562662 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 562663 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10203_solr, 127.0.0.1:10204_solr]
[junit4] 2> 562664 DEBUG (simCloudManagerPool-1029-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 562664 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 562664 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562671 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 562671 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 562672 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 562672 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 562680 DEBUG (simCloudManagerPool-1029-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 562680 DEBUG (simCloudManagerPool-1029-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 562681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 562681 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 562681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10203_solr, 127.0.0.1:10204_solr]
[junit4] 2> 562681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 562681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 562681 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 562681 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562681 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 562685 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 562687 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10205_solr
[junit4] 2> 562702 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562702 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562702 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10205_solr at time 268744464690313
[junit4] 2> 562705 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 562722 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562722 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562725 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562743 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562743 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562745 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562763 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562763 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562765 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562783 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562784 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562786 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562804 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562804 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562806 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562824 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562824 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562826 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562844 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562844 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562846 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562864 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562864 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562866 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562885 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562885 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562887 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 562905 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 562905 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 562905 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10205_solr] added at times [268744464690313], now=268754620274313
[junit4] 2> 562906 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f46bf25a6c89T7r8fx4arr81op6sa0eeakeblc",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":268744464690313,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268744464690313],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268754638476863,
[junit4] 2> "nodeNames":["127.0.0.1:10205_solr"]}}
[junit4] 2> 562908 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10205_solr
[junit4] 2> 562908 DEBUG (AutoscalingActionExecutor-1031-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f46bf25a6c89T7r8fx4arr81op6sa0eeakeblc",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":268744464690313,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268744464690313],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268754638476863,
[junit4] 2> "nodeNames":["127.0.0.1:10205_solr"]}}
[junit4] 2> 562908 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10206_solr
[junit4] 2> 563008 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563008 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563008 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 3
[junit4] 2> 563009 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10206_solr at time 268759792960213
[junit4] 2> 563029 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563029 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563029 DEBUG (ScheduledTrigger-1030-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563049 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563049 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563049 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563069 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563069 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563069 DEBUG (ScheduledTrigger-1030-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563089 DEBUG (ScheduledTrigger-1030-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563089 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563090 DEBUG (ScheduledTrigger-1030-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563108 INFO (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 563109 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 563110 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 563112 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 0
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10203_solr, 127.0.0.1:10206_solr, 127.0.0.1:10205_solr, 127.0.0.1:10204_solr]
[junit4] 2> 563113 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10203_solr, 127.0.0.1:10206_solr, 127.0.0.1:10205_solr, 127.0.0.1:10204_solr]
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=10, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10205_solr
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10206_solr
[junit4] 2> 563113 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563114 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 563114 DEBUG (ScheduledTrigger-1035-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563114 DEBUG (ScheduledTrigger-1035-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563114 DEBUG (ScheduledTrigger-1035-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563134 DEBUG (ScheduledTrigger-1035-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563134 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563134 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563154 DEBUG (ScheduledTrigger-1035-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563155 DEBUG (ScheduledTrigger-1035-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563155 DEBUG (ScheduledTrigger-1035-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563175 DEBUG (ScheduledTrigger-1035-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563175 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563175 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563195 DEBUG (ScheduledTrigger-1035-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563195 DEBUG (ScheduledTrigger-1035-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563195 DEBUG (ScheduledTrigger-1035-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563215 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563215 DEBUG (ScheduledTrigger-1035-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4 and last live nodes: 4
[junit4] 2> 563215 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 563215 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10206_solr] added at times [268759792960213], now=268770130688563
[junit4] 2> 563216 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f46f83fd42d5T7r8fx4arr81op6sa0eeakebld",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":268759792960213,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268759792960213],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268770144490463,
[junit4] 2> "nodeNames":["127.0.0.1:10206_solr"]}}
[junit4] 2> 563216 DEBUG (ScheduledTrigger-1035-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10206_solr
[junit4] 2> 563216 DEBUG (AutoscalingActionExecutor-1036-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f46f83fd42d5T7r8fx4arr81op6sa0eeakebld",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":268759792960213,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268759792960213],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268770144490463,
[junit4] 2> "nodeNames":["127.0.0.1:10206_solr"]}}
[junit4] 2> 563218 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 563218 DEBUG (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 563221 INFO (TEST-TestSimTriggerIntegration.testEventFromRestoredState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventFromRestoredState
[junit4] 2> 563224 INFO (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
[junit4] 2> 563227 DEBUG (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10207_solr
[junit4] 2> 563228 DEBUG (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 563228 DEBUG (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 563228 INFO (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 563228 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 563228 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 563229 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 563229 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 563240 DEBUG (simCloudManagerPool-1038-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563240 DEBUG (simCloudManagerPool-1038-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 563246 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10208_solr, 127.0.0.1:10207_solr]
[junit4] 2> 563247 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563248 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 563248 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 563266 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 563267 INFO (simCloudManagerPool-1038-thread-3) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestEventQueueAction instantiated
[junit4] 2> 563267 DEBUG (simCloudManagerPool-1038-thread-3) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563270 DEBUG (ScheduledTrigger-1039-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563284 DEBUG (simCloudManagerPool-1038-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563284 DEBUG (simCloudManagerPool-1038-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 563284 INFO (simCloudManagerPool-1038-thread-3) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestEventQueueAction instantiated
[junit4] 2> 563284 DEBUG (simCloudManagerPool-1038-thread-3) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563284 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 563285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10208_solr, 127.0.0.1:10207_solr]
[junit4] 2> 563285 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestEventQueueAction}], enabled=true}
[junit4] 2> 563291 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563297 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 563297 DEBUG (ScheduledTrigger-1039-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 563297 DEBUG (ScheduledTrigger-1039-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 563311 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563317 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 563317 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 563331 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563338 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 563338 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 563348 DEBUG (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10209_solr
[junit4] 2> 563351 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 563358 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 563358 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 563358 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10209_solr at time 268777265960763
[junit4] 2> 563371 DEBUG (ScheduledTrigger-1039-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 563378 DEBUG (ScheduledTrigger-1039-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 563378 DEBUG (ScheduledTrigger-1039-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 563378 DEBUG (ScheduledTrigger-1039-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10209_solr] added at times [268777265960763], now=268778285123463
[junit4] 2> 563379 DEBUG (ScheduledTrigger-1039-thread-4) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f4739576173bT7r8fx4arr81op6sa0eeakeblf",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":268777265960763,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268777265960763],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268778300747613,
[junit4] 2> "nodeNames":["127.0.0.1:10209_solr"]}}
[junit4] 2> 563391 DEBUG (ScheduledTrigger-1039-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10209_solr
[junit4] 2> 563392 INFO (AutoscalingActionExecutor-1040-thread-1) [ ] o.a.s.c.a.s.TestSimTriggerIntegration -- event: {
[junit4] 2> "id":"f4739576173bT7r8fx4arr81op6sa0eeakeblf",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":268777265960763,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268777265960763],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268778300747613,
[junit4] 2> "nodeNames":["127.0.0.1:10209_solr"]}}
[junit4] 2> 563392 INFO (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 563393 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 563407 DEBUG (AutoscalingActionExecutor-1040-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f4739576173bT7r8fx4arr81op6sa0eeakeblf",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":268777265960763,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268777265960763],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268778300747613,
[junit4] 2> "nodeNames":["127.0.0.1:10209_solr"]}}
[junit4] 2> 563407 WARN (AutoscalingActionExecutor-1040-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Unhandled exception executing actions
[junit4] 2> org.apache.solr.common.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed.
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.dequeue(ScheduledTriggers.java:583) ~[java/:?]
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$add$3(ScheduledTriggers.java:334) [java/:?]
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:515) [?:?]
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:264) [?:?]
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:209) [java/:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) [?:?]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) [?:?]
[junit4] 2> at java.lang.Thread.run(Thread.java:835) [?:?]
[junit4] 2> 563408 DEBUG (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 563408 INFO (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimCloudManager = killing node 127.0.0.1:10208_solr
[junit4] 2> 563429 DEBUG (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker: /autoscaling/nodeLost/127.0.0.1:10208_solr
[junit4] 2> 563444 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 563444 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestEventQueueAction instantiated
[junit4] 2> 563444 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563444 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 0
[junit4] 2> 563444 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 563445 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10209_solr, 127.0.0.1:10207_solr]
[junit4] 2> 563445 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10208_solr
[junit4] 2> 563445 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 - Initial livenodes: [127.0.0.1:10209_solr, 127.0.0.1:10207_solr]
[junit4] 2> 563445 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestEventQueueAction}], enabled=true}
[junit4] 2> 563445 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:10209_solr
[junit4] 2> 563445 DEBUG (ScheduledTrigger-1044-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 3
[junit4] 2> 563445 DEBUG (ScheduledTrigger-1044-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10208_solr
[junit4] 2> 563448 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563448 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 563462 DEBUG (ScheduledTrigger-1044-thread-2) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f4739576173bT7r8fx4arr81op6sa0eeakeblf",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":268777265960763,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268777265960763],
[junit4] 2> "_dequeue_time_":268781760636513,
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268778300747613,
[junit4] 2> "nodeNames":["127.0.0.1:10209_solr"],
[junit4] 2> "replaying":true}}
[junit4] 2> 563463 DEBUG (ScheduledTrigger-1044-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 563463 DEBUG (ScheduledTrigger-1044-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 563463 INFO (AutoscalingActionExecutor-1045-thread-1) [ ] o.a.s.c.a.s.TestSimTriggerIntegration -- event: {
[junit4] 2> "id":"f4739576173bT7r8fx4arr81op6sa0eeakeblf",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":268777265960763,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268777265960763],
[junit4] 2> "_dequeue_time_":268781760636513,
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268778300747613,
[junit4] 2> "nodeNames":["127.0.0.1:10209_solr"],
[junit4] 2> "replaying":true}}
[junit4] 2> 563464 DEBUG (AutoscalingActionExecutor-1045-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f4739576173bT7r8fx4arr81op6sa0eeakeblf",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":268777265960763,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268777265960763],
[junit4] 2> "_dequeue_time_":268781760636513,
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268778300747613,
[junit4] 2> "nodeNames":["127.0.0.1:10209_solr"],
[junit4] 2> "replaying":true}}
[junit4] 2> 563536 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 563537 DEBUG (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 563538 INFO (TEST-TestSimTriggerIntegration.testEventQueue-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
[junit4] 2> 563542 INFO (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTrigger
[junit4] 2> 563546 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10210_solr
[junit4] 2> 563547 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 563547 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 563547 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 563547 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 563547 INFO (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 563547 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 563547 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 563549 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10210_solr, 127.0.0.1:10211_solr]
[junit4] 2> 563550 DEBUG (simCloudManagerPool-1047-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563550 DEBUG (simCloudManagerPool-1047-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 563550 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563550 DEBUG (ScheduledTrigger-1048-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563551 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 563551 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 563551 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563551 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 563552 DEBUG (simCloudManagerPool-1047-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563552 DEBUG (simCloudManagerPool-1047-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 563552 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 563553 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563553 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_trigger - Initial livenodes: [127.0.0.1:10210_solr, 127.0.0.1:10211_solr]
[junit4] 2> 563553 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563553 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 563553 DEBUG (ScheduledTrigger-1048-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563558 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker: /autoscaling/nodeLost/127.0.0.1:10211_solr
[junit4] 2> 563570 DEBUG (ScheduledTrigger-1048-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 2
[junit4] 2> 563571 DEBUG (ScheduledTrigger-1048-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10211_solr
[junit4] 2> 563574 DEBUG (ScheduledTrigger-1048-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 and last live nodes: 2
[junit4] 2> 563574 DEBUG (ScheduledTrigger-1048-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10211_solr
[junit4] 2> 563591 DEBUG (ScheduledTrigger-1048-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 563594 DEBUG (ScheduledTrigger-1048-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 563611 DEBUG (ScheduledTrigger-1048-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 563614 DEBUG (ScheduledTrigger-1048-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 1 and last live nodes: 1
[junit4] 2> 563614 DEBUG (ScheduledTrigger-1048-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10211_solr]
[junit4] 2> 563615 DEBUG (ScheduledTrigger-1048-thread-2) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f47618d9cf99T7r8fx4arr81op6sa0eeakeblh",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":268788060245913,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268788060245913],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268790118434013,
[junit4] 2> "nodeNames":["127.0.0.1:10211_solr"]}}
[junit4] 2> 563616 DEBUG (AutoscalingActionExecutor-1049-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f47618d9cf99T7r8fx4arr81op6sa0eeakeblh",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":268788060245913,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268788060245913],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268790118434013,
[junit4] 2> "nodeNames":["127.0.0.1:10211_solr"]}}
[junit4] 2> 563639 DEBUG (simCloudManagerPool-1047-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563639 DEBUG (simCloudManagerPool-1047-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 563639 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 563640 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563640 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 563686 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 563687 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 563688 INFO (TEST-TestSimTriggerIntegration.testNodeLostTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLostTrigger
[junit4] 2> 563692 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
[junit4] 2> 563695 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10212_solr
[junit4] 2> 563696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 563696 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 563696 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 563696 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 563696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 563696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 563696 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 563701 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10212_solr, 127.0.0.1:10213_solr]
[junit4] 2> 563701 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563701 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 563701 DEBUG (ScheduledTrigger-1054-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563702 DEBUG (simCloudManagerPool-1053-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563702 DEBUG (simCloudManagerPool-1053-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 563702 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 563703 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563703 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 563703 DEBUG (simCloudManagerPool-1053-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563703 DEBUG (simCloudManagerPool-1053-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 563704 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 563704 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563704 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger - Initial livenodes: [127.0.0.1:10212_solr, 127.0.0.1:10213_solr]
[junit4] 2> 563704 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=3, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 563704 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563704 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 563704 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563704 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 563709 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- creating marker: /autoscaling/nodeAdded/127.0.0.1:10214_solr
[junit4] 2> 563722 DEBUG (ScheduledTrigger-1054-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 563725 DEBUG (ScheduledTrigger-1054-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563725 DEBUG (ScheduledTrigger-1054-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 563725 DEBUG (ScheduledTrigger-1054-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10214_solr at time 268795606028263
[junit4] 2> 563742 DEBUG (ScheduledTrigger-1054-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 563745 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563745 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 563762 DEBUG (ScheduledTrigger-1054-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 563765 DEBUG (ScheduledTrigger-1054-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563765 DEBUG (ScheduledTrigger-1054-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 563782 DEBUG (ScheduledTrigger-1054-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 3
[junit4] 2> 563786 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 563786 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 563786 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10214_solr] added at times [268795606028263], now=268798651128363
[junit4] 2> 563786 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f477da9d4fe7T7r8fx4arr81op6sa0eeakeblj",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":268795606028263,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268795606028263],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268798668464763,
[junit4] 2> "nodeNames":["127.0.0.1:10214_solr"]}}
[junit4] 2> 563786 DEBUG (ScheduledTrigger-1054-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Removing new node from tracking: 127.0.0.1:10214_solr
[junit4] 2> 563787 DEBUG (AutoscalingActionExecutor-1055-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f477da9d4fe7T7r8fx4arr81op6sa0eeakeblj",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":268795606028263,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268795606028263],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268798668464763,
[junit4] 2> "nodeNames":["127.0.0.1:10214_solr"]}}
[junit4] 2> 563788 DEBUG (simCloudManagerPool-1053-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563789 DEBUG (simCloudManagerPool-1053-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 563789 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 563789 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563789 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 563835 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 563836 DEBUG (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 563837 INFO (TEST-TestSimTriggerIntegration.testNodeAddedTrigger-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
[junit4] 2> 563841 INFO (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
[junit4] 2> 563845 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider --- new Overseer leader: 127.0.0.1:10215_solr
[junit4] 2> 563845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .auto_add_replicas and .scheduled_maintenance triggers
[junit4] 2> 563845 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 563845 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** creating new collection states, currentVersion=0
[junit4] 2> 563845 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider ** saved cluster state version 0
[junit4] 2> 563845 INFO (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.h.a.MetricsHistoryHandler No .system collection, keeping metrics history in memory.
[junit4] 2> 563846 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 0
[junit4] 2> 563846 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 563846 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger .auto_add_replicas - Initial livenodes: [127.0.0.1:10216_solr, 127.0.0.1:10215_solr]
[junit4] 2> 563846 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563846 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 563847 DEBUG (ScheduledTrigger-1060-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563847 DEBUG (simCloudManagerPool-1059-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563847 DEBUG (simCloudManagerPool-1059-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 563848 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 563848 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563848 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 563866 DEBUG (simCloudManagerPool-1059-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563866 DEBUG (simCloudManagerPool-1059-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 563866 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 563866 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563867 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10216_solr, 127.0.0.1:10215_solr]
[junit4] 2> 563867 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563867 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 563867 DEBUG (ScheduledTrigger-1060-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563867 DEBUG (ScheduledTrigger-1060-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563888 DEBUG (ScheduledTrigger-1060-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 563888 DEBUG (ScheduledTrigger-1060-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 3 and last live nodes: 2
[junit4] 2> 563889 DEBUG (TEST-TestSimTriggerIntegration.testNodeLostTriggerRestoreState-seed#[D769C3EE39CB2E8F]) [ ] o.a.s.c.a.s.SimClusterStateProvider -- created marker: /autoscaling/nodeLost/127.0.0.1:10217_solr
[junit4] 2> 563909 DEBUG (ScheduledTrigger-1060-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 and last live nodes: 3
[junit4] 2> 563909 DEBUG (ScheduledTrigger-1060-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 3
[junit4] 2> 563909 DEBUG (ScheduledTrigger-1060-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10217_solr
[junit4] 2> 563909 DEBUG (ScheduledTrigger-1060-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10217_solr
[junit4] 2> 563953 DEBUG (ScheduledTrigger-1060-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563953 DEBUG (ScheduledTrigger-1060-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563954 DEBUG (simCloudManagerPool-1059-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 563954 DEBUG (simCloudManagerPool-1059-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 563954 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 563955 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestSimTriggerIntegration TestTriggerAction init
[junit4] 2> 563955 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger node_lost_restore_trigger - Initial livenodes: [127.0.0.1:10216_solr, 127.0.0.1:10215_solr]
[junit4] 2> 563955 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- cleaning old nodeLost / nodeAdded markers
[junit4] 2> 563955 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 563955 DEBUG (ScheduledTrigger-1060-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_restore_trigger with currently live nodes: 2 and last live nodes: 2
[junit4] 2> 563955 DEBUG (ScheduledTrigger-1060-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10217_solr]
[junit4] 2> 563955 DEBUG (ScheduledTrigger-1060-thread-2) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f479febb507fT7r8fx4arr81op6sa0eeakebll",
[junit4] 2> "source":"node_lost_restore_trigger",
[junit4] 2> "eventTime":268804801908863,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268804801908863],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268807130450263,
[junit4] 2> "nodeNames":["127.0.0.1:10217_solr"]}}
[junit4] 2> 563956 DEBUG (AutoscalingActionExecutor-1061-thread-1) [ ] o.a.s.c.a.SystemLogListener Collection .system missing, skip sending event {
[junit4] 2> "id":"f479febb507fT7r8fx4arr81op6sa0eeakebll",
[junit4] 2> "source":"node_lost_restore_trigger",
[junit4] 2> "eventTime":268804801908863,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[268804801908863],
[junit4] 2> "preferredOperation":"movereplica",
[junit4] 2> "_enqueue_time_":268807130450263,
[junit4] 2> "nodeNames":["127.0.0.1:10217_solr"]}}
[junit4] 2> 563957 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Interrupted
[junit4] 2> java.lang.InterruptedException: null
[junit4] 2> at java.util.conc
[...truncated too long message...]
ng.testRepFactor1LeaderStartup(TestCloudSearcherWarming.java:117)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:567)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:835)Throwable #2: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
[junit4] > at org.apache.solr.client.solrj.impl.LBSolrClient.request(LBSolrClient.java:343)
[junit4] > at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:213)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1110)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
[junit4] > at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:207)
[junit4] > at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:224)
[junit4] > at org.apache.solr.cloud.MiniSolrCloudCluster.deleteAllCollections(MiniSolrCloudCluster.java:517)
[junit4] > at org.apache.solr.cloud.TestCloudSearcherWarming.tearDown(TestCloudSearcherWarming.java:78)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] > at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[junit4] > at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] > at java.base/java.lang.reflect.Method.invoke(Method.java:567)
[junit4] > at java.base/java.lang.Thread.run(Thread.java:835)
[junit4] 2> 2146265 INFO (SUITE-TestCloudSearcherWarming-seed#[D769C3EE39CB2E8F]-worker) [ ] o.a.s.c.ZkTestServer Shutting down ZkTestServer.
[junit4] 2> 2146266 WARN (ZkTestServer Run Thread) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2> 4 /solr/clusterprops.json
[junit4] 2> 3 /solr/collections/testRepFactor1LeaderStartup/terms/shard1
[junit4] 2> 2 /solr/configs/testRepFactor1LeaderStartup.AUTOCREATED/managed-schema
[junit4] 2> 2 /solr/configs/testRepFactor1LeaderStartup.AUTOCREATED
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/collections/testRepFactor1LeaderStartup/state.json
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/collections
[junit4] 2> 3 /solr/live_nodes
[junit4] 2>
[junit4] 2> 2146268 INFO (SUITE-TestCloudSearcherWarming-seed#[D769C3EE39CB2E8F]-worker) [ ] o.a.s.c.ZkTestServer parse host and port list: 127.0.0.1:46717
[junit4] 2> 2146268 INFO (SUITE-TestCloudSearcherWarming-seed#[D769C3EE39CB2E8F]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1 46717
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/workspace/Lucene-Solr-8.0-Linux/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudSearcherWarming_D769C3EE39CB2E8F-001
[junit4] 2> Feb 20, 2019 8:48:52 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene80): {}, docValues:{}, maxPointsInLeafNode=931, maxMBSortInHeap=6.426577818888557, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@38436af2), locale=de-BE, timezone=SystemV/EST5EDT
[junit4] 2> NOTE: Linux 4.18.0-15-generic amd64/Aleksey Shipilev 12-testing (64-bit)/cpus=8,threads=1,free=172495632,total=513277952
[junit4] 2> NOTE: All tests run in this JVM: [TestXIncludeConfig, SolrTestCaseJ4Test, TestSizeLimitedDistributedMap, DefaultValueUpdateProcessorTest, XmlUpdateRequestHandlerTest, ResponseBuilderTest, QueryResultKeyTest, TestManagedStopFilterFactory, RecoveryZkTest, ZkCLITest, TestGraphMLResponseWriter, TestSha256AuthenticationProvider, DistributedFacetPivotLargeTest, DirectSolrConnectionTest, NodeAddedTriggerTest, ByteBuffersDirectoryFactoryTest, DOMUtilTest, SolrIndexMetricsTest, UninvertDocValuesMergePolicyTest, TestStressInPlaceUpdates, QueryParsingTest, TestDocumentBuilder, TestHdfsBackupRestoreCore, TestStressRecovery, QueryElevationComponentTest, SpellCheckComponentTest, TestRealTimeGet, TestSimExtremeIndexing, TestInPlaceUpdatesStandalone, TestCodecSupport, ScriptEngineTest, TestReversedWildcardFilterFactory, TestSimpleTextCodec, PrimUtilsTest, TestFastOutputStream, ChangedSchemaMergeTest, TestDocTermOrdsUninvertLimit, TestSubQueryTransformerDistrib, TestSolrCloudWithHadoopAuthPlugin, CollectionStateFormat2Test, TestSystemIdResolver, LeaderElectionContextKeyTest, TestDynamicFieldCollectionResource, TestNumericTerms64, TestSimTriggerIntegration, ImplicitSnitchTest, TestRTimerTree, TestPerFieldSimilarity, TriLevelCompositeIdRoutingTest, TestConfigSetImmutable, TestConfigSetsAPIExclusivity, UpdateParamsTest, MoreLikeThisHandlerTest, TestSchemalessBufferedUpdates, TestTrie, BlockJoinFacetSimpleTest, ResourceLoaderTest, TestSchemaVersionResource, DistributedDebugComponentTest, TestLFUCache, TestManagedSchema, DistributedQueryComponentCustomSortTest, TestFieldCacheWithThreads, BJQParserTest, ActionThrottleTest, TestFuzzyAnalyzedSuggestions, TestCoreAdminApis, CustomCollectionTest, SortByFunctionTest, TestAuthorizationFramework, TestRestManager, StatelessScriptUpdateProcessorFactoryTest, MetricUtilsTest, TestMergePolicyConfig, TestBlendedInfixSuggestions, PeerSyncWithLeaderAndIndexFingerprintCachingTest, AlternateDirectoryTest, TestAddFieldRealTimeGet, SolrCoreMetricManagerTest, SoftAutoCommitTest, AutoscalingHistoryHandlerTest, HLLSerializationTest, TestOmitPositions, TestJsonRequest, SearchHandlerTest, OpenExchangeRatesOrgProviderTest, HdfsDirectoryTest, TestHttpShardHandlerFactory, TestTolerantUpdateProcessorCloud, TestInitParams, SuggesterFSTTest, CdcrRequestHandlerTest, TestQueryTypes, TestJoin, TestHashQParserPlugin, TestFiltering, ZkNodePropsTest, ClassificationUpdateProcessorIntegrationTest, TaggingAttributeTest, ExternalFileFieldSortTest, TestDownShardTolerantSearch, ResponseHeaderTest, SignificantTermsQParserPluginTest, TestCloudJSONFacetJoinDomain, TestSQLHandlerNonCloud, SearchRateTriggerTest, DirectUpdateHandlerTest, TestRemoteStreaming, TestOrdValues, TestSSLRandomization, DocValuesMultiTest, ReplaceNodeTest, DistributedFacetPivotSmallTest, TestBM25SimilarityFactory, AssignTest, HealthCheckHandlerTest, TestCloudInspectUtil, TestNumericRangeQuery32, TestPushWriter, ParsingFieldUpdateProcessorsTest, SolrCloudReportersTest, SolrJmxReporterCloudTest, TestCharFilters, TestPostingsSolrHighlighter, TestSolrFieldCacheBean, HdfsAutoAddReplicasIntegrationTest, TestJsonFacets, UniqFieldsUpdateProcessorFactoryTest, TestClusterStateMutator, SpellingQueryConverterTest, TestLuceneIndexBackCompat, LeaderFailoverAfterPartitionTest, LeaderVoteWaitTimeoutTest, FieldMutatingUpdateProcessorTest, AsyncCallRequestStatusResponseTest, BasicDistributedZk2Test, CleanupOldIndexTest, ConcurrentCreateRoutedAliasTest, ConfigSetsAPITest, CreateCollectionCleanupTest, CreateRoutedAliasTest, DeleteNodeTest, DeleteShardTest, DeleteStatusTest, DistribCursorPagingTest, DistributedQueueTest, DocValuesNotIndexedTest, MigrateRouteKeyTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerTest, SSLMigrationTest, SaslZkACLProviderTest, ShardRoutingCustomTest, ShardRoutingTest, SolrXmlInZkTest, SplitShardTest, TestCloudPivotFacet, TestCloudSearcherWarming]
[junit4] Completed [519/840 (2!)] on J2 in 93.84s, 2 tests, 1 error <<< FAILURES!
[...truncated 43607 lines...]
[repro] Jenkins log URL: https://jenkins.thetaphi.de/job/Lucene-Solr-8.0-Linux/199/consoleText
[repro] Revision: 29e9c8647b7cc6b2aa70452bf1c03c25302ae397
[repro] Ant options: "-Dargs=-XX:-UseCompressedOops -XX:+UseParallelGC"
[repro] Repro line: ant test -Dtestcase=TestSimTriggerIntegration -Dtests.method=testSearchRate -Dtests.seed=D769C3EE39CB2E8F -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=mr -Dtests.timezone=Asia/Tashkent -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[repro] Repro line: ant test -Dtestcase=TestCloudSearcherWarming -Dtests.method=testRepFactor1LeaderStartup -Dtests.seed=D769C3EE39CB2E8F -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=de-BE -Dtests.timezone=SystemV/EST5EDT -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[repro] ant clean
[...truncated 6 lines...]
[repro] Test suites by module:
[repro] solr/core
[repro] TestSimTriggerIntegration
[repro] TestCloudSearcherWarming
[repro] ant compile-test
[...truncated 3577 lines...]
[repro] ant test-nocompile -Dtests.dups=5 -Dtests.maxfailures=10 -Dtests.class="*.TestSimTriggerIntegration|*.TestCloudSearcherWarming" -Dtests.showOutput=onerror "-Dargs=-XX:-UseCompressedOops -XX:+UseParallelGC" -Dtests.seed=D769C3EE39CB2E8F -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=mr -Dtests.timezone=Asia/Tashkent -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[...truncated 153 lines...]
[repro] Failures:
[repro] 0/5 failed: org.apache.solr.cloud.TestCloudSearcherWarming
[repro] 0/5 failed: org.apache.solr.cloud.autoscaling.sim.TestSimTriggerIntegration
[repro] Exiting with code 0
[...truncated 43 lines...]