You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/01/14 08:52:21 UTC
[JENKINS] Lucene-Solr-Tests-7.x - Build # 314 - Still unstable
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/314/
2 tests failed.
FAILED: org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds
Error Message:
softCommit: did not find a single commit
Stack Trace:
java.lang.AssertionError: softCommit: did not find a single commit
at __randomizedtesting.SeedInfo.seed([656B5E70C9404A:5C70C5679B4B0132]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.update.SoftAutoCommitTest.assertRapidMultiCommitQueues(SoftAutoCommitTest.java:495)
at org.apache.solr.update.SoftAutoCommitTest.doTestSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:460)
at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:406)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig
Error Message:
expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}>
Stack Trace:
java.lang.AssertionError: expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}>
at __randomizedtesting.SeedInfo.seed([656B5E70C9404A:3FED68F6679FB0AD]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13196 lines...]
[junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_656B5E70C9404A-001/init-core-data-001
[junit4] 2> 2585367 WARN (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=4 numCloses=4
[junit4] 2> 2585368 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2585369 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2585369 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2585369 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 2585369 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2585480 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2585609 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.s.IndexSchema [null] Schema name=test
[junit4] 2> 2586263 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2586432 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 2586451 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 2586451 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 2586472 INFO (coreLoadExecutor-5793-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2586472 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/lib, /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/lib/classes]
[junit4] 2> 2586740 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2586817 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2587521 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 2587593 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1, trusted=true
[junit4] 2> 2587593 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 2587593 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2587593 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_656B5E70C9404A-001/init-core-data-001/]
[junit4] 2> 2587611 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=61.6044921875, floorSegmentMB=1.6728515625, forceMergeDeletesPctAllowed=11.50009028573987, segmentsPerTier=23.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2587992 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2587992 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2587993 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2587993 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2587994 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=56.1875, floorSegmentMB=0.7666015625, forceMergeDeletesPctAllowed=25.10628109939179, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 2587994 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@177a8093[collection1] main]
[junit4] 2> 2587994 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 2588065 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 2588321 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2588321 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 2588336 INFO (coreLoadExecutor-5793-thread-1) [ x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589553717211824128
[junit4] 2> 2588339 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 2588466 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 2588466 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 2588466 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 2588523 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 2588523 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 2588523 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 2588523 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 2588524 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 2588524 INFO (searcherExecutor-5794-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@177a8093[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2588525 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeDelete
[junit4] 2> 2593547 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[529 (1589553722659176448)]} 0 16
[junit4] 2> 2593549 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1589553722678050816,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2593549 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:1589553722678050816
[junit4] 2> 2593621 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d04def0[collection1] main]
[junit4] 2> 2593622 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2d04def0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.3.0):C1)))}
[junit4] 2> 2593623 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2593623 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 73
[junit4] 2> 2593636 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{delete=[529 (-1589553722755645440)]} 0 12
[junit4] 2> 2594146 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2594158 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1cf69982[collection1] main]
[junit4] 2> 2594158 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2594161 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1cf69982[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2594182 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[550 (1589553723341799424)]} 0 0
[junit4] 2> 2594683 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2594697 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@687f28f5[collection1] main]
[junit4] 2> 2594699 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@687f28f5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.3.0):C1)))}
[junit4] 2> 2594699 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2594836 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2594836 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2594836 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@73a54a78[collection1] realtime]
[junit4] 2> 2594836 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2600858 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxTimeDelete
[junit4] 2> 2600867 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testHardCommitWithinAndSoftCommitMaxTimeRapidAdds
[junit4] 2> 2605923 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5000 (1589553735609090048)]} 0 42
[junit4] 2> 2605924 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5001 (1589553735653130240)]} 0 0
[junit4] 2> 2605924 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5002 (1589553735654178816)]} 0 0
[junit4] 2> 2605924 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5003 (1589553735654178817)]} 0 0
[junit4] 2> 2605924 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5004 (1589553735654178818)]} 0 0
[junit4] 2> 2605924 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxTimeMillis: 370904234868575ns - 370904191411969ns == 43ms
[junit4] 2> 2605925 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxExpectedSoft=1
[junit4] 2> 2605925 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxExpectedHard=1
[junit4] 2> 2606432 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2606485 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@7b51341d[collection1] main]
[junit4] 2> 2606505 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7b51341d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.3.0):C1) Uninverting(_2(7.3.0):C5)))}
[junit4] 2> 2606508 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2607125 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2607125 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2607125 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@6556bc04[collection1] realtime]
[junit4] 2> 2607125 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2612623 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testHardCommitWithinAndSoftCommitMaxTimeRapidAdds
[junit4] 2> 2612639 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxDocs
[junit4] 2> 2620653 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8000 (1589553751097606144)]} 0 0
[junit4] 2> 2620653 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8001 (1589553751098654720)]} 0 0
[junit4] 2> 2620653 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8002 (1589553751098654721)]} 0 0
[junit4] 2> 2620654 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8003 (1589553751098654722)]} 0 0
[junit4] 2> 2620654 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8004 (1589553751099703296)]} 0 0
[junit4] 2> 2620654 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8005 (1589553751099703297)]} 0 0
[junit4] 2> 2620654 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8006 (1589553751099703298)]} 0 0
[junit4] 2> 2620654 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[8007 (1589553751099703299)]} 0 0
[junit4] 2> 2620655 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2620656 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2620702 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3adfd973[collection1] main]
[junit4] 2> 2620704 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3adfd973[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.3.0):C1) Uninverting(_2(7.3.0):C5) Uninverting(_3(7.3.0):C8)))}
[junit4] 2> 2620704 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2620709 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2620709 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@32b49ccc[collection1] realtime]
[junit4] 2> 2620709 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2621710 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxDocs-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxDocs
[junit4] 2> 2621712 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSoftCommitWithinAndHardCommitMaxTimeDelete
[junit4] 2> 2626724 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[529 (1589553757463511040)]} 0 0
[junit4] 2> 2626724 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1589553757464559616,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2626724 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:1589553757464559616
[junit4] 2> 2626745 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@58fb9cda[collection1] main]
[junit4] 2> 2626747 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@58fb9cda[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.3.0):C1) Uninverting(_2(7.3.0):C5) Uninverting(_3(7.3.0):C8) Uninverting(_4(7.3.0):C1)))}
[junit4] 2> 2626747 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2626747 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 22
[junit4] 2> 2626760 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{delete=[529 (-1589553757488676864)]} 0 12
[junit4] 2> 2627270 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2627271 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@576d7989[collection1] main]
[junit4] 2> 2627271 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2627272 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@576d7989[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.3.0):C1) Uninverting(_2(7.3.0):C5) Uninverting(_3(7.3.0):C8)))}
[junit4] 2> 2627273 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[550 (1589553758040227840)]} 0 0
[junit4] 2> 2627781 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2627783 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@3b126b7c[collection1] main]
[junit4] 2> 2627783 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2627818 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3b126b7c[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_2(7.3.0):C5) Uninverting(_3(7.3.0):C8) Uninverting(_5(7.3.0):C1)))}
[junit4] 2> 2627963 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2627963 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2627963 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@14ec21a9[collection1] realtime]
[junit4] 2> 2627963 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2633964 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSoftCommitWithinAndHardCommitMaxTimeDelete
[junit4] 2> 2633986 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSoftCommitWithinAndHardCommitMaxTimeRapidAdds
[junit4] 2> 2638987 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5000 (1589553770322198528)]} 0 0
[junit4] 2> 2638987 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5001 (1589553770323247104)]} 0 0
[junit4] 2> 2638988 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5002 (1589553770324295680)]} 0 0
[junit4] 2> 2638988 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5003 (1589553770324295681)]} 0 0
[junit4] 2> 2638988 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5004 (1589553770324295682)]} 0 0
[junit4] 2> 2638988 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxTimeMillis: 370937298646319ns - 370937296600499ns == 2ms
[junit4] 2> 2638988 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxExpectedSoft=1
[junit4] 2> 2638988 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxExpectedHard=1
[junit4] 2> 2639487 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2639490 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@493f71ee[collection1] main]
[junit4] 2> 2639490 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2639491 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@493f71ee[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_5(7.3.0):C1) Uninverting(_6(7.3.0):C5)))}
[junit4] 2> 2640199 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2640199 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2640199 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@79fc1d9f[collection1] realtime]
[junit4] 2> 2640199 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2645598 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSoftCommitWithinAndHardCommitMaxTimeRapidAdds
[junit4] 2> 2645600 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSoftCommitWithinAndHardCommitMaxTimeMixedAdds
[junit4] 2> 2654027 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[529 (1589553786092781568)]} 0 0
[junit4] 2> 2654536 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2654565 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@358fe68[collection1] main]
[junit4] 2> 2654567 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@358fe68[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_5(7.3.0):C1) Uninverting(_6(7.3.0):C5) Uninverting(_7(7.3.0):C1)))}
[junit4] 2> 2654567 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2655227 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2655227 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2655228 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@242e5de9[collection1] realtime]
[junit4] 2> 2655228 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2655228 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[530 (1589553787352121344)]} 0 0
[junit4] 2> 2655728 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2655731 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@4c78e4d3[collection1] main]
[junit4] 2> 2655731 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2655733 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4c78e4d3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_5(7.3.0):C1) Uninverting(_6(7.3.0):C5) Uninverting(_7(7.3.0):C1) Uninverting(_8(7.3.0):C1)))}
[junit4] 2> 2656428 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2656428 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2656429 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@63b404c2[collection1] realtime]
[junit4] 2> 2656429 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2657432 INFO (TEST-SoftAutoCommitTest.testSoftCommitWithinAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSoftCommitWithinAndHardCommitMaxTimeMixedAdds
[junit4] 2> 2657434 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testHardCommitWithinAndSoftCommitMaxTimeDelete
[junit4] 2> 2662440 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[529 (1589553794915500032)]} 0 0
[junit4] 2> 2662441 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.DirectUpdateHandler2 start commit{_version_=1589553794916548608,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2662441 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:1589553794916548608
[junit4] 2> 2662497 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d24e93[collection1] main]
[junit4] 2> 2662497 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2662510 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2d24e93[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_5(7.3.0):C1) Uninverting(_6(7.3.0):C5) Uninverting(_8(7.3.0):C1) Uninverting(_9(7.3.0):C1)))}
[junit4] 2> 2662510 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{commit=} 0 69
[junit4] 2> 2662511 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{delete=[529 (-1589553794988900352)]} 0 0
[junit4] 2> 2663015 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2663015 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@4f74e31a[collection1] main]
[junit4] 2> 2663016 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2663017 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4f74e31a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_5(7.3.0):C1) Uninverting(_6(7.3.0):C5) Uninverting(_8(7.3.0):C1)))}
[junit4] 2> 2663018 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[550 (1589553795520528384)]} 0 0
[junit4] 2> 2663529 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2663532 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@ade2aca[collection1] main]
[junit4] 2> 2663532 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2663566 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@ade2aca[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_6(7.3.0):C5) Uninverting(_8(7.3.0):C1) Uninverting(_a(7.3.0):C1)))}
[junit4] 2> 2663720 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2663720 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2663720 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@731784d8[collection1] realtime]
[junit4] 2> 2663721 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2669724 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeDelete-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testHardCommitWithinAndSoftCommitMaxTimeDelete
[junit4] 2> 2669725 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeMixedAdds
[junit4] 2> 2678142 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[529 (1589553811379191808)]} 0 0
[junit4] 2> 2678642 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2678645 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@4cd003c5[collection1] main]
[junit4] 2> 2678645 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2678647 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4cd003c5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_6(7.3.0):C5) Uninverting(_8(7.3.0):C1) Uninverting(_a(7.3.0):C1) Uninverting(_b(7.3.0):C1)))}
[junit4] 2> 2679342 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2679342 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2679344 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[530 (1589553812639580160)]} 0 0
[junit4] 2> 2679346 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@cc6b57e[collection1] realtime]
[junit4] 2> 2679346 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2679851 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2679851 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@71518fa8[collection1] main]
[junit4] 2> 2679851 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2679853 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@71518fa8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_6(7.3.0):C5) Uninverting(_a(7.3.0):C1) Uninverting(_b(7.3.0):C1) Uninverting(_c(7.3.0):C1)))}
[junit4] 2> 2680560 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2680560 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2680560 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@52592ae0[collection1] realtime]
[junit4] 2> 2680560 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2681560 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxTimeMixedAdds
[junit4] 2> 2681562 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeRapidAdds
[junit4] 2> 2686564 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5000 (1589553820210298880)]} 0 0
[junit4] 2> 2686564 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5001 (1589553820211347456)]} 0 0
[junit4] 2> 2686564 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5002 (1589553820211347457)]} 0 0
[junit4] 2> 2686565 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5003 (1589553820211347458)]} 0 0
[junit4] 2> 2686565 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[5004 (1589553820212396032)]} 0 0
[junit4] 2> 2686565 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxTimeMillis: 370984875347707ns - 370984873291891ns == 2ms
[junit4] 2> 2686565 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxExpectedSoft=1
[junit4] 2> 2686565 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.SoftAutoCommitTest maxExpectedHard=1
[junit4] 2> 2687065 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2689610 INFO (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxTimeRapidAdds
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeRapidAdds -Dtests.seed=656B5E70C9404A -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=sk-SK -Dtests.timezone=Africa/Juba -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 8.05s J0 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds <<<
[junit4] > Throwable #1: java.lang.AssertionError: softCommit: did not find a single commit
[junit4] > at __randomizedtesting.SeedInfo.seed([656B5E70C9404A:5C70C5679B4B0132]:0)
[junit4] > at org.apache.solr.update.SoftAutoCommitTest.assertRapidMultiCommitQueues(SoftAutoCommitTest.java:495)
[junit4] > at org.apache.solr.update.SoftAutoCommitTest.doTestSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:460)
[junit4] > at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds(SoftAutoCommitTest.java:406)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2689612 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2689629 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testHardCommitWithinAndSoftCommitMaxTimeMixedAdds
[junit4] 2> 2689648 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@40b9d218[collection1] main]
[junit4] 2> 2689650 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@40b9d218[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_a(7.3.0):C1) Uninverting(_b(7.3.0):C1) Uninverting(_c(7.3.0):C1) Uninverting(_d(7.3.0):C5)))}
[junit4] 2> 2689650 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2689658 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2689680 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[529 (1589553823456690176)]} 0 20
[junit4] 2> 2689683 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@328ad852[collection1] realtime]
[junit4] 2> 2689683 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2690181 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2690181 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@f4d66e9[collection1] main]
[junit4] 2> 2690181 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2690183 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@f4d66e9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_a(7.3.0):C1) Uninverting(_c(7.3.0):C1) Uninverting(_d(7.3.0):C5) Uninverting(_e(7.3.0):C1)))}
[junit4] 2> 2690890 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2690890 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2690891 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@1505a966[collection1] realtime]
[junit4] 2> 2690891 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2690892 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=null path=null params={}{add=[530 (1589553824748535808)]} 0 0
[junit4] 2> 2691392 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
[junit4] 2> 2691394 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@57dbd6a8[collection1] main]
[junit4] 2> 2691395 INFO (commitScheduler-5798-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2691396 INFO (searcherExecutor-5794-thread-1) [ ] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@57dbd6a8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_3(7.3.0):C8) Uninverting(_a(7.3.0):C1) Uninverting(_d(7.3.0):C5) Uninverting(_e(7.3.0):C1) Uninverting(_f(7.3.0):C1)))}
[junit4] 2> 2692092 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=false,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 2692092 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@1522ef1c commitCommandVersion:0
[junit4] 2> 2692093 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.s.SolrIndexSearcher Opening [Searcher@318f171f[collection1] realtime]
[junit4] 2> 2692093 INFO (commitScheduler-5797-thread-1) [ ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 2693092 INFO (TEST-SoftAutoCommitTest.testHardCommitWithinAndSoftCommitMaxTimeMixedAdds-seed#[656B5E70C9404A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testHardCommitWithinAndSoftCommitMaxTimeMixedAdds
[junit4] 2> 2693093 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> 2693093 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=122317935
[junit4] 2> 2693093 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2693093 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@390f8054: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6ab61c11
[junit4] 2> 2693096 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2693096 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1954a22f: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@204b9bc9
[junit4] 2> 2693098 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2693098 INFO (SUITE-SoftAutoCommitTest-seed#[656B5E70C9404A]-worker) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@609e7690: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@56cf770a
[junit4] 2> 2693099 INFO (coreCloseExecutor-5799-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5230c1b
[junit4] 2> 2693099 INFO (coreCloseExecutor-5799-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=86182939
[junit4] 2> 2693099 INFO (coreCloseExecutor-5799-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@690d5eaf: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@eadfb1b
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.update.SoftAutoCommitTest_656B5E70C9404A-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {range_facet_l_dv=PostingsFormat(name=Direct), multiDefault=BlockTreeOrds(blocksize=128), subject=BlockTreeOrds(blocksize=128), intDefault=FST50, id_i1=BlockTreeOrds(blocksize=128), id=PostingsFormat(name=Direct), range_facet_i_dv=FST50, text=Lucene50(blocksize=128), range_facet_l=FST50, timestamp=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), _version_=DocValuesFormat(name=Lucene70), intDefault=DocValuesFormat(name=Lucene70), id_i1=DocValuesFormat(name=Memory), range_facet_i_dv=DocValuesFormat(name=Lucene70), intDvoDefault=DocValuesFormat(name=Direct), range_facet_l=DocValuesFormat(name=Lucene70), timestamp=DocValuesFormat(name=Lucene70)}, maxPointsInLeafNode=100, maxMBSortInHeap=6.29229350801232, sim=RandomSimilarity(queryNorm=true): {}, locale=sk-SK, timezone=Africa/Juba
[junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=361537432,total=530579456
[junit4] 2> NOTE: All tests run in this JVM: [SolrCoreTest, BigEndianAscendingWordSerializerTest, SolrMetricManagerTest, TestLegacyField, BaseCdcrDistributedZkTest, TestRebalanceLeaders, TestSort, TestSortByMinMaxFunction, TestShortCircuitedRequests, SolrGangliaReporterTest, MetricTriggerTest, CacheHeaderTest, FullHLLTest, TestMultiWordSynonyms, TestPullReplicaErrorHandling, TestPHPSerializedResponseWriter, TestCorePropertiesReload, UninvertDocValuesMergePolicyTest, ConnectionManagerTest, TestRecovery, TermsComponentTest, AnalysisAfterCoreReloadTest, TestDistributedStatsComponentCardinality, ResourceLoaderTest, TestSolrCloudSnapshots, TestHighFrequencyDictionaryFactory, DistributedQueryElevationComponentTest, CleanupOldIndexTest, VMParamsZkACLAndCredentialsProvidersTest, TestBinaryField, DistributedFacetPivotSmallTest, TestImplicitCoreProperties, StressHdfsTest, CollectionsAPISolrJTest, TestJsonFacets, TestXmlQParserPlugin, TestNumericRangeQuery64, InfixSuggestersTest, TestSubQueryTransformerDistrib, DateMathParserTest, HdfsTlogReplayBufferedWhileIndexingTest, SimpleFacetsTest, TestCloudPseudoReturnFields, BasicAuthStandaloneTest, JsonLoaderTest, LeaderInitiatedRecoveryOnShardRestartTest, TestRTimerTree, TestPivotHelperCode, CursorPagingTest, MinimalSchemaTest, TolerantUpdateProcessorTest, SearchHandlerTest, TestOnReconnectListenerSupport, FileUtilsTest, NoCacheHeaderTest, SolrPluginUtilsTest, ResponseLogComponentTest, TestLeaderElectionWithEmptyReplica, TestReversedWildcardFilterFactory, DistributedFacetPivotLargeTest, TestDistributedSearch, ParsingFieldUpdateProcessorsTest, TestStressUserVersions, TestCustomSort, TestChildDocTransformer, TestHighlightDedupGrouping, ConjunctionSolrSpellCheckerTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestNRTOpen, SliceStateTest, TestEmbeddedSolrServerSchemaAPI, TestNumericTerms64, JvmMetricsTest, TestMergePolicyConfig, TestConfig, TestSQLHandlerNonCloud, TestLargeCluster, TestReplicationHandler, TestPhraseSuggestions, SchemaApiFailureTest, TestClusterProperties, TestSolrDeletionPolicy1, TestConfigSetImmutable, TestCodecSupport, URLClassifyProcessorTest, DeleteInactiveReplicaTest, TestSuggestSpellingConverter, TestRandomFaceting, TestGraphTermsQParserPlugin, CollectionStateFormat2Test, OverseerModifyCollectionTest, DocumentBuilderTest, TestExactStatsCache, MissingSegmentRecoveryTest, TestPKIAuthenticationPlugin, TestConfigsApi, TestSolrCloudWithDelegationTokens, UpdateRequestProcessorFactoryTest, BasicDistributedZkTest, SmileWriterTest, TestSystemCollAutoCreate, TestExceedMaxTermLength, TestLocalFSCloudBackupRestore, TestIndexingPerformance, TestSolrCoreParser, TestDistribStateManager, TestCryptoKeys, TestRestManager, TestLegacyNumericRangeQueryBuilder, TermVectorComponentTest, DistribCursorPagingTest, SimpleCollectionCreateDeleteTest, CurrencyFieldTypeTest, CloneFieldUpdateProcessorFactoryTest, DeleteReplicaTest, CursorMarkTest, TestFoldingMultitermQuery, SpellCheckCollatorTest, SpellCheckComponentTest, UtilsToolTest, AnalysisErrorHandlingTest, PeerSyncTest, TestDelegationWithHadoopAuth, AssignBackwardCompatibilityTest, CachingDirectoryFactoryTest, AutoscalingHistoryHandlerTest, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, ShardRoutingTest, TestRandomDVFaceting, TestGroupingSearch, BadIndexSchemaTest, TestFiltering, TestBadConfig, SoftAutoCommitTest]
[junit4] Completed [442/766 (1!)] on J0 in 107.86s, 10 tests, 1 failure <<< FAILURES!
[...truncated 409 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_656B5E70C9404A-001/init-core-data-001
[junit4] 2> 3382055 WARN (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
[junit4] 2> 3382055 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 3382056 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3382056 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3382056 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.a.s.TestClusterStateProvider ####### Using simulated components? true
[junit4] 2> 3382057 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 3 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_656B5E70C9404A-001/tempDir-001
[junit4] 2> 3382057 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3382057 INFO (Thread-4822) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3382057 INFO (Thread-4822) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3382058 ERROR (Thread-4822) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3382157 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:45210
[junit4] 2> 3382158 INFO (zkConnectionManagerCallback-4242-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382160 INFO (jetty-launcher-4239-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3382160 INFO (jetty-launcher-4239-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3382160 INFO (jetty-launcher-4239-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3382161 INFO (jetty-launcher-4239-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1ea202bb{/solr,null,AVAILABLE}
[junit4] 2> 3382166 INFO (jetty-launcher-4239-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75dbae46{/solr,null,AVAILABLE}
[junit4] 2> 3382167 INFO (jetty-launcher-4239-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@2166895d{HTTP/1.1,[http/1.1]}{127.0.0.1:43279}
[junit4] 2> 3382167 INFO (jetty-launcher-4239-thread-1) [ ] o.e.j.s.Server Started @3392169ms
[junit4] 2> 3382167 INFO (jetty-launcher-4239-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43279}
[junit4] 2> 3382167 ERROR (jetty-launcher-4239-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3382167 INFO (jetty-launcher-4239-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 3382167 INFO (jetty-launcher-4239-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3382167 INFO (jetty-launcher-4239-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3382167 INFO (jetty-launcher-4239-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T08:09:08.484Z
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@155e280a{/solr,null,AVAILABLE}
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3eac0b5e{HTTP/1.1,[http/1.1]}{127.0.0.1:39984}
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-2) [ ] o.e.j.s.Server Started @3392173ms
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1df32f70{HTTP/1.1,[http/1.1]}{127.0.0.1:41867}
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=39984}
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.e.j.s.Server Started @3392173ms
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41867}
[junit4] 2> 3382171 ERROR (jetty-launcher-4239-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3382171 ERROR (jetty-launcher-4239-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T08:09:08.488Z
[junit4] 2> 3382171 INFO (jetty-launcher-4239-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-14T08:09:08.488Z
[junit4] 2> 3382173 INFO (zkConnectionManagerCallback-4248-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382173 INFO (jetty-launcher-4239-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3382176 INFO (zkConnectionManagerCallback-4244-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382176 INFO (zkConnectionManagerCallback-4246-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382176 INFO (jetty-launcher-4239-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3382176 INFO (jetty-launcher-4239-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3382179 INFO (jetty-launcher-4239-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45210/solr
[junit4] 2> 3382180 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f3b70e170001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3382180 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f3b70e170003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3382180 INFO (zkConnectionManagerCallback-4252-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382181 INFO (jetty-launcher-4239-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45210/solr
[junit4] 2> 3382181 INFO (jetty-launcher-4239-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:45210/solr
[junit4] 2> 3382182 INFO (zkConnectionManagerCallback-4257-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382182 INFO (zkConnectionManagerCallback-4260-thread-1-processing-n:127.0.0.1:41867_solr) [n:127.0.0.1:41867_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382182 INFO (zkConnectionManagerCallback-4262-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382183 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f3b70e170005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3382183 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160f3b70e170007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3382184 INFO (zkConnectionManagerCallback-4266-thread-1-processing-n:127.0.0.1:39984_solr) [n:127.0.0.1:39984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382184 INFO (zkConnectionManagerCallback-4264-thread-1-processing-n:127.0.0.1:43279_solr) [n:127.0.0.1:43279_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382246 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3382247 INFO (jetty-launcher-4239-thread-3) [n:127.0.0.1:41867_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3382247 INFO (jetty-launcher-4239-thread-2) [n:127.0.0.1:39984_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3382247 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43279_solr
[junit4] 2> 3382247 INFO (jetty-launcher-4239-thread-3) [n:127.0.0.1:41867_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41867_solr
[junit4] 2> 3382247 INFO (jetty-launcher-4239-thread-2) [n:127.0.0.1:39984_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39984_solr
[junit4] 2> 3382247 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.c.Overseer Overseer (id=99347159343104008-127.0.0.1:43279_solr-n_0000000000) starting
[junit4] 2> 3382254 INFO (zkCallback-4259-thread-1-processing-n:127.0.0.1:41867_solr) [n:127.0.0.1:41867_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3382262 INFO (zkCallback-4263-thread-1-processing-n:127.0.0.1:43279_solr) [n:127.0.0.1:43279_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3382262 INFO (zkCallback-4265-thread-1-processing-n:127.0.0.1:39984_solr) [n:127.0.0.1:39984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3382267 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43279_solr
[junit4] 2> 3382274 INFO (zkCallback-4265-thread-1-processing-n:127.0.0.1:39984_solr) [n:127.0.0.1:39984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3382274 INFO (zkCallback-4259-thread-1-processing-n:127.0.0.1:41867_solr) [n:127.0.0.1:41867_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3382274 INFO (zkCallback-4263-thread-1-processing-n:127.0.0.1:43279_solr) [n:127.0.0.1:43279_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3382295 INFO (jetty-launcher-4239-thread-3) [n:127.0.0.1:41867_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41867.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382295 INFO (jetty-launcher-4239-thread-2) [n:127.0.0.1:39984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39984.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382302 INFO (jetty-launcher-4239-thread-3) [n:127.0.0.1:41867_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41867.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382302 INFO (jetty-launcher-4239-thread-2) [n:127.0.0.1:39984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39984.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382302 INFO (jetty-launcher-4239-thread-3) [n:127.0.0.1:41867_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41867.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382302 INFO (jetty-launcher-4239-thread-2) [n:127.0.0.1:39984_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_39984.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382303 INFO (jetty-launcher-4239-thread-2) [n:127.0.0.1:39984_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_656B5E70C9404A-001/tempDir-001/node2/.
[junit4] 2> 3382303 INFO (jetty-launcher-4239-thread-3) [n:127.0.0.1:41867_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_656B5E70C9404A-001/tempDir-001/node3/.
[junit4] 2> 3382305 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43279.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382312 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43279.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382312 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43279.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1013ebd8
[junit4] 2> 3382313 INFO (jetty-launcher-4239-thread-1) [n:127.0.0.1:43279_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_656B5E70C9404A-001/tempDir-001/node1/.
[junit4] 2> 3382370 INFO (zkConnectionManagerCallback-4274-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382372 INFO (zkConnectionManagerCallback-4278-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3382372 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3382373 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:45210/solr ready
[junit4] 2> 3382375 INFO (qtp1278225779-20064) [n:127.0.0.1:39984_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=1&replicationFactor=2&name=.system&nrtReplicas=2&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3382379 INFO (OverseerThreadFactory-8121-thread-1-processing-n:127.0.0.1:43279_solr) [n:127.0.0.1:43279_solr ] o.a.s.c.CreateCollectionCmd Create collection .system
[junit4] 2> 3382483 INFO (OverseerStateUpdate-99347159343104008-127.0.0.1:43279_solr-n_0000000000) [n:127.0.0.1:43279_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":".system_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:39984/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3382485 INFO (OverseerStateUpdate-99347159343104008-127.0.0.1:43279_solr-n_0000000000) [n:127.0.0.1:43279_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":".system",
[junit4] 2> "shard":"shard1",
[junit4] 2> "cor
[...truncated too long message...]
4] 2> 3386417 INFO (coreCloseExecutor-8158-thread-1) [n:127.0.0.1:43279_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.c.SolrCore [.system_shard1_replica_p4] CLOSING SolrCore org.apache.solr.core.SolrCore@1c095872
[junit4] 2> 3386417 INFO (coreCloseExecutor-8156-thread-1) [n:127.0.0.1:39984_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3237df86: rootName = solr_39984, domain = solr.core..system.shard1.replica_n1, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@58078cc8
[junit4] 2> 3386417 INFO (coreCloseExecutor-8158-thread-1) [n:127.0.0.1:43279_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_p4, tag=470374514
[junit4] 2> 3386418 INFO (coreCloseExecutor-8158-thread-1) [n:127.0.0.1:43279_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@13d20061: rootName = solr_43279, domain = solr.core..system.shard1.replica_p4, service url = null, agent id = null] for registry solr.core..system.shard1.replica_p4 / com.codahale.metrics.MetricRegistry@1f500ef6
[junit4] 2> 3386418 INFO (zkCallback-4285-thread-2-processing-n:127.0.0.1:46045_solr) [n:127.0.0.1:46045_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 3386418 INFO (jetty-closer-4240-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7dfe8c1f{/solr,null,UNAVAILABLE}
[junit4] 2> 3386425 INFO (coreCloseExecutor-8158-thread-1) [n:127.0.0.1:43279_solr c:.system s:shard1 r:core_node6 x:.system_shard1_replica_p4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=470374514
[junit4] 2> 3386425 INFO (coreCloseExecutor-8156-thread-1) [n:127.0.0.1:39984_solr c:.system s:shard1 r:core_node3 x:.system_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=133602386
[junit4] 2> 3386435 INFO (jetty-closer-4240-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99347159343104008-127.0.0.1:43279_solr-n_0000000000) closing
[junit4] 2> 3386435 INFO (OverseerStateUpdate-99347159343104008-127.0.0.1:43279_solr-n_0000000000) [n:127.0.0.1:43279_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43279_solr
[junit4] 2> 3386435 WARN (OverseerAutoScalingTriggerThread-99347159343104008-127.0.0.1:43279_solr-n_0000000000) [n:127.0.0.1:43279_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 3386436 INFO (jetty-closer-4240-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@75dbae46{/solr,null,UNAVAILABLE}
[junit4] 2> 3386437 INFO (jetty-closer-4240-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1ea202bb{/solr,null,UNAVAILABLE}
[junit4] 2> 3386437 ERROR (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3386437 INFO (SUITE-TestClusterStateProvider-seed#[656B5E70C9404A]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45210 45210
[junit4] 2> 3386495 INFO (Thread-4822) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:45210 45210
[junit4] 2> 3386496 WARN (Thread-4822) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 3 /solr/configs/.system
[junit4] 2> 3 /solr/configs/.system/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 20 /solr/collections/.system/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 2 /solr/overseer_elect/election/99347159343104008-127.0.0.1:43279_solr-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_656B5E70C9404A-001
[junit4] 2> Jan 14, 2018 8:09:12 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=false): {}, locale=ja-JP, timezone=Asia/Magadan
[junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=81116544,total=518520832
[junit4] 2> NOTE: All tests run in this JVM: [SolrCoreTest, BigEndianAscendingWordSerializerTest, SolrMetricManagerTest, TestLegacyField, BaseCdcrDistributedZkTest, TestRebalanceLeaders, TestSort, TestSortByMinMaxFunction, TestShortCircuitedRequests, SolrGangliaReporterTest, MetricTriggerTest, CacheHeaderTest, FullHLLTest, TestMultiWordSynonyms, TestPullReplicaErrorHandling, TestPHPSerializedResponseWriter, TestCorePropertiesReload, UninvertDocValuesMergePolicyTest, ConnectionManagerTest, TestRecovery, TermsComponentTest, AnalysisAfterCoreReloadTest, TestDistributedStatsComponentCardinality, ResourceLoaderTest, TestSolrCloudSnapshots, TestHighFrequencyDictionaryFactory, DistributedQueryElevationComponentTest, CleanupOldIndexTest, VMParamsZkACLAndCredentialsProvidersTest, TestBinaryField, DistributedFacetPivotSmallTest, TestImplicitCoreProperties, StressHdfsTest, CollectionsAPISolrJTest, TestJsonFacets, TestXmlQParserPlugin, TestNumericRangeQuery64, InfixSuggestersTest, TestSubQueryTransformerDistrib, DateMathParserTest, HdfsTlogReplayBufferedWhileIndexingTest, SimpleFacetsTest, TestCloudPseudoReturnFields, BasicAuthStandaloneTest, JsonLoaderTest, LeaderInitiatedRecoveryOnShardRestartTest, TestRTimerTree, TestPivotHelperCode, CursorPagingTest, MinimalSchemaTest, TolerantUpdateProcessorTest, SearchHandlerTest, TestOnReconnectListenerSupport, FileUtilsTest, NoCacheHeaderTest, SolrPluginUtilsTest, ResponseLogComponentTest, TestLeaderElectionWithEmptyReplica, TestReversedWildcardFilterFactory, DistributedFacetPivotLargeTest, TestDistributedSearch, ParsingFieldUpdateProcessorsTest, TestStressUserVersions, TestCustomSort, TestChildDocTransformer, TestHighlightDedupGrouping, ConjunctionSolrSpellCheckerTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestNRTOpen, SliceStateTest, TestEmbeddedSolrServerSchemaAPI, TestNumericTerms64, JvmMetricsTest, TestMergePolicyConfig, TestConfig, TestSQLHandlerNonCloud, TestLargeCluster, TestReplicationHandler, TestPhraseSuggestions, SchemaApiFailureTest, TestClusterProperties, TestSolrDeletionPolicy1, TestConfigSetImmutable, TestCodecSupport, URLClassifyProcessorTest, DeleteInactiveReplicaTest, TestSuggestSpellingConverter, TestRandomFaceting, TestGraphTermsQParserPlugin, CollectionStateFormat2Test, OverseerModifyCollectionTest, DocumentBuilderTest, TestExactStatsCache, MissingSegmentRecoveryTest, TestPKIAuthenticationPlugin, TestConfigsApi, TestSolrCloudWithDelegationTokens, UpdateRequestProcessorFactoryTest, BasicDistributedZkTest, SmileWriterTest, TestSystemCollAutoCreate, TestExceedMaxTermLength, TestLocalFSCloudBackupRestore, TestIndexingPerformance, TestSolrCoreParser, TestDistribStateManager, TestCryptoKeys, TestRestManager, TestLegacyNumericRangeQueryBuilder, TermVectorComponentTest, DistribCursorPagingTest, SimpleCollectionCreateDeleteTest, CurrencyFieldTypeTest, CloneFieldUpdateProcessorFactoryTest, DeleteReplicaTest, CursorMarkTest, TestFoldingMultitermQuery, SpellCheckCollatorTest, SpellCheckComponentTest, UtilsToolTest, AnalysisErrorHandlingTest, PeerSyncTest, TestDelegationWithHadoopAuth, AssignBackwardCompatibilityTest, CachingDirectoryFactoryTest, AutoscalingHistoryHandlerTest, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest, ShardRoutingTest, TestRandomDVFaceting, TestGroupingSearch, BadIndexSchemaTest, TestFiltering, TestBadConfig, SoftAutoCommitTest, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, TestSolrQueryParser, IndexSchemaTest, ReturnFieldsTest, TestAnalyzedSuggestions, TestComponentsName, SOLR749Test, TestSolrIndexConfig, SolrIndexConfigTest, NumericFieldsTest, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, TestDFRSimilarityFactory, TestIBSimilarityFactory, ScriptEngineTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestDocumentBuilder, SystemInfoHandlerTest, DistributedMLTComponentTest, TestLuceneIndexBackCompat, ChaosMonkeyNothingIsSafeWithPullReplicasTest, LegacyCloudClusterPropTest, MoveReplicaHDFSFailoverTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, OverseerTaskQueueTest, RestartWhileUpdatingTest, SSLMigrationTest, SaslZkACLProviderTest, SolrXmlInZkTest, TestCloudInspectUtil, TestCloudSearcherWarming, TestConfigSetsAPI, TestDownShardTolerantSearch, TestLockTree, TestMiniSolrCloudClusterSSL, TestRandomRequestDistribution, TestRequestForwarding, TestSolrCloudWithSecureImpersonation, TestStressCloudBlindAtomicUpdates, TestTlogReplica, HttpTriggerListenerTest, NodeAddedTriggerTest, SearchRateTriggerTest, TestClusterStateProvider]
[junit4] Completed [573/766 (2!)] on J0 in 4.71s, 2 tests, 1 failure <<< FAILURES!
[...truncated 48792 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 320 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/320/
5 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestBlobHandler
Error Message:
38 threads leaked from SUITE scope at org.apache.solr.handler.TestBlobHandler: 1) Thread[id=6903, name=qtp258116223-6903, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:149) at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:199) at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:306) at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:246) at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:224) at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177) at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736) at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717) at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380) at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139) at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595) at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564) at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141) at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132) at org.eclipse.jetty.server.Server.handle(Server.java:530) at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347) at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256) at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279) at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102) at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.io.ManagedSelector$$Lambda$24/1133556656.run(Unknown Source) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=6914, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=6909, name=qtp258116223-6909, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=6943, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-EventThread, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 5) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=6944, name=OverseerStateUpdate-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415) at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=6906, name=qtp258116223-6906, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=6935, name=zkCallback-883-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=6941, name=zkCallback-883-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=6968, name=updateExecutor-879-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=6925, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=6947, name=ScheduledTrigger-2312-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=6976, name=zkCallback-883-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=6905, name=qtp258116223-6905, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=6945, name=OverseerCollectionConfigSetProcessor-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339) at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392) at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=6942, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-SendThread(127.0.0.1:42967), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 17) Thread[id=6908, name=qtp258116223-6908, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=6902, name=qtp258116223-6902, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=6922, name=zkCallback-883-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=6959, name=ScheduledTrigger-2312-thread-3, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=6930, name=watches-885-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=6969, name=recoveryExecutor-880-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=6950, name=coreZkRegister-2300-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=6932, name=searcherExecutor-2306-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=6904, name=qtp258116223-6904-acceptor-0@360e97d4-ServerConnector@3fe8a307{HTTP/1.1,[http/1.1]}{127.0.0.1:36328}, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=6915, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722) 34) Thread[id=6949, name=OverseerThreadFactory-2314-thread-1, state=WAITING, group=Overseer collection creation process.] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70) at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:315) at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:57) at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:203) at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:544) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=6946, name=OverseerAutoScalingTriggerThread-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=WAITING, group=Overseer autoscaling triggers] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 38 threads leaked from SUITE scope at org.apache.solr.handler.TestBlobHandler:
1) Thread[id=6903, name=qtp258116223-6903, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Object.wait(Native Method)
at org.apache.solr.cloud.OverseerTaskQueue$LatchWatcher.await(OverseerTaskQueue.java:149)
at org.apache.solr.cloud.OverseerTaskQueue.offer(OverseerTaskQueue.java:199)
at org.apache.solr.handler.admin.CollectionsHandler.handleResponse(CollectionsHandler.java:306)
at org.apache.solr.handler.admin.CollectionsHandler.invokeAction(CollectionsHandler.java:246)
at org.apache.solr.handler.admin.CollectionsHandler.handleRequestBody(CollectionsHandler.java:224)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
at org.eclipse.jetty.server.Server.handle(Server.java:530)
at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.io.ManagedSelector$$Lambda$24/1133556656.run(Unknown Source)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=6914, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=6909, name=qtp258116223-6909, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=6943, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-EventThread, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
5) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=6944, name=OverseerStateUpdate-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer state updater.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.amILeader(Overseer.java:415)
at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:145)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=6906, name=qtp258116223-6906, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=6935, name=zkCallback-883-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=6941, name=zkCallback-883-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=6968, name=updateExecutor-879-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=6925, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=6947, name=ScheduledTrigger-2312-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=6976, name=zkCallback-883-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=6905, name=qtp258116223-6905, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=6945, name=OverseerCollectionConfigSetProcessor-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=TIMED_WAITING, group=Overseer collection creation process.]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:339)
at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:392)
at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:193)
at java.lang.Thread.run(Thread.java:748)
16) Thread[id=6942, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-SendThread(127.0.0.1:42967), state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
17) Thread[id=6908, name=qtp258116223-6908, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:600)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:49)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:663)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=6902, name=qtp258116223-6902, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=6922, name=zkCallback-883-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=6959, name=ScheduledTrigger-2312-thread-3, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=6930, name=watches-885-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=6969, name=recoveryExecutor-880-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=6950, name=coreZkRegister-2300-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=6932, name=searcherExecutor-2306-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=6904, name=qtp258116223-6904-acceptor-0@360e97d4-ServerConnector@3fe8a307{HTTP/1.1,[http/1.1]}{127.0.0.1:36328}, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:379)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:638)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=6915, name=Connection evictor, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler]
at java.lang.Object.wait(Native Method)
at java.lang.Object.wait(Object.java:502)
at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722)
34) Thread[id=6949, name=OverseerThreadFactory-2314-thread-1, state=WAITING, group=Overseer collection creation process.]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at java.lang.Thread.sleep(Native Method)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:315)
at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:57)
at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:203)
at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:544)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=6946, name=OverseerAutoScalingTriggerThread-99373310955421706-127.0.0.1:36328_wv%2Fx-n_0000000002, state=WAITING, group=Overseer autoscaling triggers]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:166)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([D9ED50908F29A796]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestBlobHandler
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=6909, name=qtp258116223-6909, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=6906, name=qtp258116223-6906, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=6935, name=zkCallback-883-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=6941, name=zkCallback-883-thread-5, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=6968, name=updateExecutor-879-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=6947, name=ScheduledTrigger-2312-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=6976, name=zkCallback-883-thread-7, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=6905, name=qtp258116223-6905, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=6942, name=zkConnectionManagerCallback-884-thread-1-processing-n:127.0.0.1:36328_wv%2Fx-SendThread(127.0.0.1:42967), state=TIMED_WAITING, group=TGRP-TestBlobHandler] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:997) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1060) 11) Thread[id=6902, name=qtp258116223-6902, state=RUNNABLE, group=TGRP-TestBlobHandler] at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140) at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=6922, name=zkCallback-883-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=6959, name=ScheduledTrigger-2312-thread-3, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=6930, name=watches-885-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=6969, name=recoveryExecutor-880-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 16) Thread[id=6950, name=coreZkRegister-2300-thread-1, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=6932, name=searcherExecutor-2306-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler] at java.lang.Object.wait(Native Method) at java.lang.Object.wait(Object.java:502) at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722) 25) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=6909, name=qtp258116223-6909, state=RUNNABLE, group=TGRP-TestBlobHandler]
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:375)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:304)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:179)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=6975, name=zkCallback-883-thread-6, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concu
[...truncated too long message...]
ask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 18) Thread[id=6907, name=qtp258116223-6907, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 19) Thread[id=6967, name=searcherExecutor-2315-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 20) Thread[id=6921, name=zkConnectionManagerCallback-884-thread-1, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 21) Thread[id=6926, name=Scheduler-1674259162, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 22) Thread[id=6965, name=ScheduledTrigger-2312-thread-4, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 23) Thread[id=6910, name=Scheduler-411863045, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 24) Thread[id=6923, name=Thread-2986, state=WAITING, group=TGRP-TestBlobHandler]
[junit4] > at java.lang.Object.wait(Native Method)
[junit4] > at java.lang.Object.wait(Object.java:502)
[junit4] > at org.apache.solr.core.CloserThread.run(CoreContainer.java:1722)
[junit4] > 25) Thread[id=6936, name=zkCallback-883-thread-3, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 26) Thread[id=6948, name=ScheduledTrigger-2312-thread-2, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 27) Thread[id=6937, name=zkCallback-883-thread-4, state=TIMED_WAITING, group=TGRP-TestBlobHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([D9ED50908F29A796]:0)
[junit4] Completed [263/767 (3!)] on J2 in 109.37s, 1 test, 3 errors <<< FAILURES!
[...truncated 49708 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 319 - Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/319/
6 tests failed.
FAILED: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test
Error Message:
Timed out waiting for replica core_node54 (1516235137951) to replicate from leader core_node46 (0)
Stack Trace:
java.lang.AssertionError: Timed out waiting for replica core_node54 (1516235137951) to replicate from leader core_node46 (0)
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC:CBD057E1E472EC34]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForReplicationFromReplicas(AbstractFullDistribZkTestBase.java:2143)
at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest.test(ChaosMonkeyNothingIsSafeWithPullReplicasTest.java:268)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest: 1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1414, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 5) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=1416, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501) 9) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest:
1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1414, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
5) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=1416, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-EventThread, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:501)
9) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051) 4) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=1789, name=zkCallback-292-thread-4, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=1746, name=zkCallback-292-thread-2, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=1415, name=TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]-SendThread(127.0.0.1:40492), state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1051)
4) Thread[id=1417, name=zkConnectionManagerCallback-293-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=1747, name=zkCallback-292-thread-3, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=1920, name=zkCallback-292-thread-5, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=1419, name=zkCallback-292-thread-1, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeWithPullReplicasTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC]:0)
FAILED: org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory
Error Message:
expected:<8> but was:<9>
Stack Trace:
java.lang.AssertionError: expected:<8> but was:<9>
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC:2E78CCC6F0C67ECB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.handler.admin.AutoscalingHistoryHandlerTest.testHistory(AutoscalingHistoryHandlerTest.java:259)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test
Error Message:
Expected no errors: [{type=ADD,id=1,message=Unexpected type of routeKey: null}]
Stack Trace:
java.lang.AssertionError: Expected no errors: [{type=ADD,id=1,message=Unexpected type of routeKey: null}]
at __randomizedtesting.SeedInfo.seed([4384683B4A8E81CC:CBD057E1E472EC34]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.assertUpdateResponse(TimeRoutedAliasUpdateProcessorTest.java:270)
at org.apache.solr.update.processor.TimeRoutedAliasUpdateProcessorTest.test(TimeRoutedAliasUpdateProcessorTest.java:132)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.common.cloud.TestCollectionStateWatchers.testPredicateFailureTimesOut
Error Message:
Watchers for collection should be removed after timeout
Stack Trace:
java.lang.AssertionError: Watchers for collection should be removed after timeout
at __randomizedtesting.SeedInfo.seed([9EDBA3692A4378C6:63E2C555735BADBC]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.common.cloud.TestCollectionStateWatchers.waitFor(TestCollectionStateWatchers.java:107)
at org.apache.solr.common.cloud.TestCollectionStateWatchers.testPredicateFailureTimesOut(TestCollectionStateWatchers.java:216)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11872 lines...]
[junit4] Suite: org.apache.solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/init-core-data-001
[junit4] 2> 657073 WARN (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 657074 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 657110 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
[junit4] 2> 657110 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 657110 INFO (SUITE-ChaosMonkeyNothingIsSafeWithPullReplicasTest-seed#[4384683B4A8E81CC]-worker) [ ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /oi/pl
[junit4] 2> 657111 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ChaosMonkeyNothingIsSafeWithPullReplicasTest Starting ChaosMonkey test with 2 shards and 8 nodes
[junit4] 2> 657231 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 657232 INFO (Thread-235) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 657232 INFO (Thread-235) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 657234 ERROR (Thread-235) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 657332 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkTestServer start zk server on port:40492
[junit4] 2> 657333 INFO (zkConnectionManagerCallback-273-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 657459 INFO (zkConnectionManagerCallback-275-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 657470 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
[junit4] 2> 657481 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
[junit4] 2> 657490 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
[junit4] 2> 657491 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
[junit4] 2> 657510 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
[junit4] 2> 657511 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
[junit4] 2> 657512 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
[junit4] 2> 657513 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
[junit4] 2> 657552 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
[junit4] 2> 657559 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
[junit4] 2> 657566 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
[junit4] 2> 657695 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase Will use NRT replicas unless explicitly asked otherwise
[junit4] 2> 658691 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 658758 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 658758 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 658759 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 658759 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@66c7a66a{/oi/pl,null,AVAILABLE}
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@51d464c2{HTTP/1.1,[http/1.1]}{127.0.0.1:45771}
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server Started @686706ms
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/tempDir-001/control/data, replicaType=NRT, hostContext=/oi/pl, hostPort=45771, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores}
[junit4] 2> 658769 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 658769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-18T00:22:31.182Z
[junit4] 2> 658887 INFO (zkConnectionManagerCallback-277-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 658889 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 658889 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/solr.xml
[junit4] 2> 658911 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 658911 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 658912 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 658942 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40492/solr
[junit4] 2> 658967 INFO (zkConnectionManagerCallback-281-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 659058 INFO (zkConnectionManagerCallback-283-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 659769 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 659770 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45771_oi%2Fpl
[junit4] 2> 659770 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.Overseer Overseer (id=99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) starting
[junit4] 2> 659898 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45771_oi%2Fpl
[junit4] 2> 659941 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 662044 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 662354 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 662354 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 662357 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores
[junit4] 2> 662786 INFO (zkConnectionManagerCallback-289-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 662806 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 662807 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40492/solr ready
[junit4] 2> 662809 INFO (qtp656415825-1370) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45771_oi%252Fpl&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 662931 INFO (OverseerThreadFactory-519-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.CreateCollectionCmd Create collection control_collection
[junit4] 2> 663403 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 663404 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 663518 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 664866 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 665139 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema [control_collection_shard1_replica_n1] Schema name=test
[junit4] 2> 666504 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 666910 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'control_collection_shard1_replica_n1' using configuration from collection control_collection, trusted=true
[junit4] 2> 666910 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.control_collection.shard1.replica_n1' (registry 'solr.core.control_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 666911 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 666911 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [[control_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores/control_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/control-001/cores/control_collection_shard1_replica_n1/data/]
[junit4] 2> 666914 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 667101 WARN (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 668049 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 668049 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 668051 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 668052 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 668095 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=38.2451171875, floorSegmentMB=1.5830078125, forceMergeDeletesPctAllowed=21.032870952662417, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 668214 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@64667f4[control_collection_shard1_replica_n1] main]
[junit4] 2> 668224 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 668226 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 668227 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 668247 INFO (searcherExecutor-522-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl x:control_collection_shard1_replica_n1 s:shard1 c:control_collection) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SolrCore [control_collection_shard1_replica_n1] Registered new searcher Searcher@64667f4[control_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 668248 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589887590110068736
[junit4] 2> 668328 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 668328 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 668328 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:45771/oi/pl/control_collection_shard1_replica_n1/
[junit4] 2> 668329 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 668329 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:45771/oi/pl/control_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 668329 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 668356 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 668387 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:45771/oi/pl/control_collection_shard1_replica_n1/ shard1
[junit4] 2> 668387 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 668426 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl c:control_collection s:shard1 x:control_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&newCollection=true&name=control_collection_shard1_replica_n1&action=CREATE&numShards=1&collection=control_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=5023
[junit4] 2> 668472 INFO (qtp656415825-1370) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 668490 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/control_collection/state.json] for collection [control_collection] has occurred - updating... (live nodes size: [1])
[junit4] 2> 669472 INFO (qtp656415825-1370) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=control_collection&nrtReplicas=1&action=CREATE&numShards=1&createNodeSet=127.0.0.1:45771_oi%252Fpl&wt=javabin&version=2} status=0 QTime=6664
[junit4] 2> 669954 INFO (zkConnectionManagerCallback-293-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 669955 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 669957 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:40492/solr ready
[junit4] 2> 669957 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ChaosMonkey monkey: init - expire sessions:false cause connection loss:false
[junit4] 2> 669963 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 670304 INFO (OverseerThreadFactory-519-thread-2-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.CreateCollectionCmd Create collection collection1
[junit4] 2> 670346 INFO (OverseerCollectionConfigSetProcessor-99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 670386 WARN (OverseerThreadFactory-519-thread-2-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.CreateCollectionCmd It is unusual to create a collection (collection1) without cores.
[junit4] 2> 670620 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 670620 INFO (qtp656415825-1376) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf1&name=collection1&nrtReplicas=1&action=CREATE&numShards=2&createNodeSet=&stateFormat=1&wt=javabin&version=2} status=0 QTime=657
[junit4] 2> 671948 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 1 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001 of type NRT
[junit4] 2> 671963 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 672094 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 672094 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@29ee3f0f{/oi/pl,null,AVAILABLE}
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4d8bf561{HTTP/1.1,[http/1.1]}{127.0.0.1:36274}
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server Started @700033ms
[junit4] 2> 672095 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/tempDir-001/jetty1, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/oi/pl, hostPort=36274, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores}
[junit4] 2> 672096 ERROR (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 672096 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-18T00:22:44.509Z
[junit4] 2> 672330 INFO (zkConnectionManagerCallback-295-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 672338 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
[junit4] 2> 672338 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/solr.xml
[junit4] 2> 672342 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 672342 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 672398 INFO (OverseerCollectionConfigSetProcessor-99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 672423 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 672543 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40492/solr
[junit4] 2> 672666 INFO (zkConnectionManagerCallback-299-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 672676 INFO (zkConnectionManagerCallback-301-thread-1-processing-n:127.0.0.1:36274_oi%2Fpl) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 672918 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 672920 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 672939 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36274_oi%2Fpl
[junit4] 2> 672939 INFO (zkCallback-282-thread-1-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 672940 INFO (zkCallback-292-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 673178 INFO (zkCallback-300-thread-1-processing-n:127.0.0.1:36274_oi%2Fpl) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 674377 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 674605 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 674605 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 674628 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores
[junit4] 2> 675095 INFO (qtp308972610-1421) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :addreplica with params node=127.0.0.1:36274_oi%252Fpl&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 676167 INFO (OverseerThreadFactory-519-thread-3-processing-n:127.0.0.1:45771_oi%2Fpl) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.a.c.AddReplicaCmd Node Identified 127.0.0.1:36274_oi%2Fpl for creating new replica
[junit4] 2> 676624 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 676625 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 677981 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 678179 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema [collection1_shard2_replica_n41] Schema name=test
[junit4] 2> 680660 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.IndexSchema Loaded schema test/1.6 with uniqueid field id
[junit4] 2> 681019 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard2_replica_n41' using configuration from collection collection1, trusted=true
[junit4] 2> 681020 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1.shard2.replica_n41' (registry 'solr.core.collection1.shard2.replica_n41') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@149ea79b
[junit4] 2> 681020 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 681020 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [[collection1_shard2_replica_n41] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores/collection1_shard2_replica_n41], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-1-001/cores/collection1_shard2_replica_n41/data/]
[junit4] 2> 681099 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 681802 WARN (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 682154 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 682154 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 682155 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Hard AutoCommit: if uncommited for 15000ms;
[junit4] 2> 682155 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 682156 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=38.2451171875, floorSegmentMB=1.5830078125, forceMergeDeletesPctAllowed=21.032870952662417, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 682218 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.SolrIndexSearcher Opening [Searcher@16ac7a2a[collection1_shard2_replica_n41] main]
[junit4] 2> 682838 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
[junit4] 2> 682845 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
[junit4] 2> 682846 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 682846 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589887605417181184
[junit4] 2> 682855 INFO (searcherExecutor-533-thread-1-processing-n:127.0.0.1:36274_oi%2Fpl x:collection1_shard2_replica_n41 s:shard2 c:collection1) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SolrCore [collection1_shard2_replica_n41] Registered new searcher Searcher@16ac7a2a[collection1_shard2_replica_n41] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36274/oi/pl/collection1_shard2_replica_n41/
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.SyncStrategy http://127.0.0.1:36274/oi/pl/collection1_shard2_replica_n41/ has no replicas
[junit4] 2> 682879 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 682896 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36274/oi/pl/collection1_shard2_replica_n41/ shard2
[junit4] 2> 683222 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 683223 INFO (qtp308972610-1425) [n:127.0.0.1:36274_oi%2Fpl c:collection1 s:shard2 x:collection1_shard2_replica_n41] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=conf1&name=collection1_shard2_replica_n41&action=CREATE&collection=collection1&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=6599
[junit4] 2> 683247 INFO (qtp308972610-1421) [n:127.0.0.1:36274_oi%2Fpl ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={node=127.0.0.1:36274_oi%252Fpl&action=ADDREPLICA&collection=collection1&shard=shard2&type=NRT&wt=javabin&version=2} status=0 QTime=8151
[junit4] 2> 683755 INFO (OverseerCollectionConfigSetProcessor-99367973660000260-127.0.0.1:45771_oi%2Fpl-n_0000000000) [n:127.0.0.1:45771_oi%2Fpl ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 686605 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.AbstractFullDistribZkTestBase create jetty 2 in directory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/shard-2-001 of type NRT
[junit4] 2> 686691 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T22:27:37+01:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 686710 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 686710 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@41a87c3d{/oi/pl,null,AVAILABLE}
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5b9afbdd{HTTP/1.1,[http/1.1]}{127.0.0.1:40228}
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.e.j.s.Server Started @714649ms
[junit4] 2> 686711 INFO (TEST-ChaosMonkeyNothingIsSafeWithPullReplicasTest.test-seed#[4384683B4A8E81CC]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.ChaosMonkeyNothingIsSafeWithPullReplicasTest_4384683B4A8E81CC-001/tempDir-001/jetty2, replicaType=NRT, solrconfig=solrconfig.xml, hostContext=/oi/pl, hostPort=40228, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/L
[...truncated too long message...]
20 INFO (coreCloseExecutor-161-thread-2) [n:127.0.0.1:58516_solr c:delayed s:shard1 r:core_node2 x:delayed_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.delayed.shard1.leader, tag=813847959
[junit4] 2> 407320 INFO (coreCloseExecutor-161-thread-3) [n:127.0.0.1:58516_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard1.replica_n1, tag=1164830252
[junit4] 2> 407320 INFO (coreCloseExecutor-161-thread-3) [n:127.0.0.1:58516_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@61b12365: rootName = solr_58516, domain = solr.core.testcollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testcollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@39cab994
[junit4] 2> 407340 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.c.SolrCore [falsepredicate_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@2912a4ca
[junit4] 2> 407340 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.falsepredicate.shard1.replica_n1, tag=689087690
[junit4] 2> 407340 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@19d5a472: rootName = solr_59305, domain = solr.core.falsepredicate.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.falsepredicate.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@21105b4d
[junit4] 2> 407432 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.c.SolrCore [waitforstate_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3efc472e
[junit4] 2> 407510 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.waitforstate.shard1.replica_n1, tag=1056720686
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-1) [n:127.0.0.1:59305_solr c:falsepredicate s:shard1 r:core_node2 x:falsepredicate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.falsepredicate.shard1.leader, tag=689087690
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.c.SolrCore [testcollection_shard3_replica_n4] CLOSING SolrCore org.apache.solr.core.SolrCore@12043cdc
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testcollection.shard3.replica_n4, tag=302267612
[junit4] 2> 407510 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5fd5a81c: rootName = solr_59305, domain = solr.core.testcollection.shard3.replica_n4, service url = null, agent id = null] for registry solr.core.testcollection.shard3.replica_n4 / com.codahale.metrics.MetricRegistry@24e2995d
[junit4] 2> 407728 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7b53ffd8: rootName = solr_58516, domain = solr.core.waitforstate.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.waitforstate.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@2722b1e
[junit4] 2> 407865 INFO (coreCloseExecutor-162-thread-2) [n:127.0.0.1:59305_solr c:testcollection s:shard3 r:core_node7 x:testcollection_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard3.leader, tag=302267612
[junit4] 2> 407865 INFO (coreCloseExecutor-161-thread-3) [n:127.0.0.1:58516_solr c:testcollection s:shard1 r:core_node3 x:testcollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testcollection.shard1.leader, tag=1164830252
[junit4] 2> 407866 INFO (coreCloseExecutor-161-thread-4) [n:127.0.0.1:58516_solr c:waitforstate s:shard1 r:core_node2 x:waitforstate_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.waitforstate.shard1.leader, tag=1056720686
[junit4] 2> 407964 INFO (jetty-closer-13-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7a14c812{/solr,null,UNAVAILABLE}
[junit4] 2> 407980 INFO (jetty-closer-13-thread-3) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409428 INFO (jetty-closer-13-thread-4) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@400b5127{/solr,null,UNAVAILABLE}
[junit4] 2> 409429 INFO (jetty-closer-13-thread-4) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409476 INFO (jetty-closer-13-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99368291327213580-127.0.0.1:59305_solr-n_0000000003) closing
[junit4] 2> 409476 INFO (OverseerStateUpdate-99368291327213580-127.0.0.1:59305_solr-n_0000000003) [n:127.0.0.1:59305_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:59305_solr
[junit4] 2> 409496 INFO (jetty-closer-13-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4e60539c{/solr,null,UNAVAILABLE}
[junit4] 2> 409495 WARN (OverseerAutoScalingTriggerThread-99368291327213580-127.0.0.1:59305_solr-n_0000000003) [n:127.0.0.1:59305_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 409503 INFO (jetty-closer-13-thread-2) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409509 INFO (jetty-closer-13-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@33f3b0b0{/solr,null,UNAVAILABLE}
[junit4] 2> 409510 INFO (jetty-closer-13-thread-1) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 409674 ERROR (SUITE-TestCollectionStateWatchers-seed#[9EDBA3692A4378C6]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 409674 INFO (SUITE-TestCollectionStateWatchers-seed#[9EDBA3692A4378C6]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35127 35127
[junit4] 2> 409801 INFO (Thread-53) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35127 35127
[junit4] 2> 409802 WARN (Thread-53) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/aliases.json
[junit4] 2> 8 /solr/clusterprops.json
[junit4] 2> 7 /solr/security.json
[junit4] 2> 7 /solr/configs/config
[junit4] 2> 4 /solr/collections/stateformat1/state.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 57 /solr/collections/falsepredicate/state.json
[junit4] 2> 57 /solr/collections/testcollection/state.json
[junit4] 2> 17 /solr/collections/waitforstate/state.json
[junit4] 2> 8 /solr/clusterstate.json
[junit4] 2> 7 /solr/collections/currentstate/state.json
[junit4] 2> 7 /solr/collections/delayed/state.json
[junit4] 2> 6 /solr/collections/tobedeleted/state.json
[junit4] 2> 5 /solr/collections/stateformat1/state.json
[junit4] 2> 3 /solr/overseer_elect/election/99368291327213578-127.0.0.1:36015_solr-n_0000000000
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 8 /solr/live_nodes
[junit4] 2> 8 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-solrj/test/J2/temp/solr.common.cloud.TestCollectionStateWatchers_9EDBA3692A4378C6-001
[junit4] 2> Jan 18, 2018 1:45:26 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 6 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1400, maxMBSortInHeap=6.309672053689489, sim=RandomSimilarity(queryNorm=true): {}, locale=zh-CN, timezone=US/Mountain
[junit4] 2> NOTE: Linux 3.13.0-88-generic amd64/Oracle Corporation 1.8.0_144 (64-bit)/cpus=4,threads=1,free=171145704,total=313524224
[junit4] 2> NOTE: All tests run in this JVM: [SolrExampleXMLTest, TestValidatingJsonMap, HyperbolicSineEvaluatorTest, TestSolrJErrorHandling, AddEvaluatorTest, CloudSolrClientBuilderTest, StreamExpressionToExplanationTest, TestCollectionStateWatchers]
[junit4] Completed [35/146 (1!)] on J2 in 131.29s, 9 tests, 1 failure <<< FAILURES!
[...truncated 47567 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 318 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/318/
9 tests failed.
FAILED: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.testRandom
Error Message:
Error from server at http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection: {"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection":8}
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection: {"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection":8}
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:60F8CCF7EF65F933]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.assertFacetCountsAreCorrect(TestCloudJSONFacetJoinDomain.java:461)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.assertFacetCountsAreCorrect(TestCloudJSONFacetJoinDomain.java:465)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.assertFacetCountsAreCorrect(TestCloudJSONFacetJoinDomain.java:429)
at org.apache.solr.cloud.TestCloudJSONFacetJoinDomain.testRandom(TestCloudJSONFacetJoinDomain.java:370)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit
Error Message:
Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
Stack Trace:
java.lang.AssertionError: Shard split did not succeed after a previous failed split attempt left sub-shards in construction state
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:EBF97A57627002CA]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.api.collections.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:297)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestGenericDistributedQueue.testDistributedQueue
Error Message:
Test abandoned because suite timeout was reached.
Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.sim.TestGenericDistributedQueue
Error Message:
Suite timeout exceeded (>= 7200000 msec).
Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40]:0)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode
Error Message:
did not finish processing changes
Stack Trace:
java.lang.AssertionError: did not finish processing changes
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:B55BF45B9148C058]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:297)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling
Error Message:
Both triggers should have fired by now
Stack Trace:
java.lang.AssertionError: Both triggers should have fired by now
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:E99641DD8CAFACD2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testTriggerThrottling(TestTriggerIntegration.java:171)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventFromRestoredState
Error Message:
The trigger did not fire at all
Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:12825D387708EB2A]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventFromRestoredState(TestTriggerIntegration.java:673)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState
Error Message:
Two TriggerAction instances should have been created by now
Stack Trace:
java.lang.AssertionError: Two TriggerAction instances should have been created by now
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:9A89608764C5AEED]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:361)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown
Error Message:
[CapturedEvent{timestamp=16445112123730749, stage=SUCCEEDED, actionName='null', event={ "id":"3a6cba6973b2c7T746dhdwn6gduim68xvnj3bn16", "source":"node_added_cooldown_trigger", "eventTime":16445096538124999, "eventType":"NODEADDED", "properties":{ "eventTimes":[16445096538124999], "_enqueue_time_":16445109634069299, "nodeNames":["127.0.0.1:10012_solr"]}}, context={}, config={ "trigger":"node_added_cooldown_trigger", "afterAction":[], "stage":[ "FAILED", "SUCCEEDED", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener", "beforeAction":[]}, message='null'}]
Stack Trace:
java.lang.AssertionError: [CapturedEvent{timestamp=16445112123730749, stage=SUCCEEDED, actionName='null', event={
"id":"3a6cba6973b2c7T746dhdwn6gduim68xvnj3bn16",
"source":"node_added_cooldown_trigger",
"eventTime":16445096538124999,
"eventType":"NODEADDED",
"properties":{
"eventTimes":[16445096538124999],
"_enqueue_time_":16445109634069299,
"nodeNames":["127.0.0.1:10012_solr"]}}, context={}, config={
"trigger":"node_added_cooldown_trigger",
"afterAction":[],
"stage":[
"FAILED",
"SUCCEEDED",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
"beforeAction":[]}, message='null'}]
at __randomizedtesting.SeedInfo.seed([12B4E9F85E054F40:230A841C20AF3AB2]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown(TestTriggerIntegration.java:1100)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12255 lines...]
[junit4] Suite: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain
[junit4] 2> 1693017 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/init-core-data-001
[junit4] 2> 1693018 WARN (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=30 numCloses=30
[junit4] 2> 1693043 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 1693058 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 1693059 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001
[junit4] 2> 1693059 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1693106 INFO (Thread-2750) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1693106 INFO (Thread-2750) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1693198 ERROR (Thread-2750) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1693217 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:55238
[junit4] 2> 1693249 INFO (zkConnectionManagerCallback-1217-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693359 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T11:27:37-10:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T11:27:37-10:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 1693360 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@61d95de{/solr,null,AVAILABLE}
[junit4] 2> 1693391 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3d778931{HTTP/1.1,[http/1.1]}{127.0.0.1:48892}
[junit4] 2> 1693391 INFO (jetty-launcher-1214-thread-1) [ ] o.e.j.s.Server Started @1717137ms
[junit4] 2> 1693391 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=48892}
[junit4] 2> 1693391 ERROR (jetty-launcher-1214-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1693392 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-17T11:50:02.413Z
[junit4] 2> 1693599 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 1693599 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 1693599 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@59346517{/solr,null,AVAILABLE}
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6e94e9c4{HTTP/1.1,[http/1.1]}{127.0.0.1:35017}
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.e.j.s.Server Started @1717424ms
[junit4] 2> 1693678 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=35017}
[junit4] 2> 1693679 ERROR (jetty-launcher-1214-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1693679 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-17T11:50:02.700Z
[junit4] 2> 1693828 INFO (zkConnectionManagerCallback-1219-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693828 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1693870 INFO (zkConnectionManagerCallback-1221-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1693870 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1693926 INFO (jetty-launcher-1214-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55238/solr
[junit4] 2> 1694380 INFO (zkConnectionManagerCallback-1225-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694398 INFO (jetty-launcher-1214-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:55238/solr
[junit4] 2> 1694399 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x16103f45e900003, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1694480 INFO (zkConnectionManagerCallback-1229-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694481 INFO (zkConnectionManagerCallback-1231-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1694698 INFO (zkConnectionManagerCallback-1233-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1695360 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1695360 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:35017_solr
[junit4] 2> 1695361 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.Overseer Overseer (id=99365014872260614-127.0.0.1:35017_solr-n_0000000000) starting
[junit4] 2> 1695486 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:35017_solr
[junit4] 2> 1695500 INFO (OverseerStateUpdate-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1695734 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1695748 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1695769 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48892_solr
[junit4] 2> 1695878 INFO (zkCallback-1232-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1696018 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1696717 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1696901 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1696901 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1696910 INFO (jetty-launcher-1214-thread-2) [n:127.0.0.1:35017_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node2/.
[junit4] 2> 1697032 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1697191 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1697191 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1697229 INFO (jetty-launcher-1214-thread-1) [n:127.0.0.1:48892_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node1/.
[junit4] 2> 1698466 INFO (zkConnectionManagerCallback-1239-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1698550 INFO (zkConnectionManagerCallback-1243-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1698551 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1698553 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:55238/solr ready
[junit4] 2> 1700460 INFO (qtp67429917-6510) [n:127.0.0.1:35017_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1700468 INFO (OverseerThreadFactory-1838-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection
[junit4] 2> 1700698 INFO (OverseerStateUpdate-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:35017/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1700716 INFO (OverseerStateUpdate-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:48892/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1703316 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node4&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3&action=CREATE&numShards=2&shard=shard2&wt=javabin
[junit4] 2> 1703316 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1703468 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node2&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin
[junit4] 2> 1703468 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1704242 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704399 INFO (zkCallback-1232-thread-1-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704402 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704403 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1704554 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1704724 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.IndexSchema [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] Schema name=example
[junit4] 2> 1706312 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1706531 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.IndexSchema [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] Schema name=example
[junit4] 2> 1706930 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema example/1.6 with uniqueid field id
[junit4] 2> 1707023 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1' using configuration from collection org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection, trusted=true
[junit4] 2> 1707024 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_35017.solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard1.replica_n1' (registry 'solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1707024 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1707024 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrCore [[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node2/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node2/./org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/data/]
[junit4] 2> 1707130 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=91.4140625, floorSegmentMB=1.3056640625, forceMergeDeletesPctAllowed=9.912107369120065, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18030994669460867
[junit4] 2> 1707172 WARN (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1707332 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.IndexSchema Loaded schema example/1.6 with uniqueid field id
[junit4] 2> 1707599 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.CoreContainer Creating SolrCore 'org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3' using configuration from collection org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection, trusted=true
[junit4] 2> 1707600 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_48892.solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard2.replica_n3' (registry 'solr.core.org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection.shard2.replica_n3') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@3e7a86c4
[junit4] 2> 1707600 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1707600 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrCore [[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node1/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.TestCloudJSONFacetJoinDomain_12B4E9F85E054F40-001/tempDir-001/node1/./org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/data/]
[junit4] 2> 1707730 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=46, maxMergeAtOnceExplicit=15, maxMergedSegmentMB=91.4140625, floorSegmentMB=1.3056640625, forceMergeDeletesPctAllowed=9.912107369120065, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18030994669460867
[junit4] 2> 1708343 WARN (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.RequestHandlers INVALID paramSet a in requestHandler {type = requestHandler,name = /dump,class = DumpRequestHandler,attributes = {initParams=a, name=/dump, class=DumpRequestHandler},args = {defaults={a=A,b=B}}}
[junit4] 2> 1708508 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1708508 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1708515 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1708515 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1708516 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=89.4091796875, floorSegmentMB=1.666015625, forceMergeDeletesPctAllowed=29.841986508731416, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1708538 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2215244e[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] main]
[junit4] 2> 1708586 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1708587 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1708587 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1708588 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589840254189174784
[junit4] 2> 1708615 INFO (searcherExecutor-1843-thread-1-processing-n:127.0.0.1:35017_solr x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1 s:shard1 c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection r:core_node2) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SolrCore [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] Registered new searcher Searcher@2215244e[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1708617 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1708617 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1708617 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:35017/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/
[junit4] 2> 1708618 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1708618 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:35017/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/ has no replicas
[junit4] 2> 1708618 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1708636 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:35017/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1/ shard1
[junit4] 2> 1708647 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708647 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708647 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708690 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1708711 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node2&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1&action=CREATE&numShards=2&shard=shard1&wt=javabin} status=0 QTime=5243
[junit4] 2> 1708849 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708858 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708858 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1708908 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1708908 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1708909 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1708909 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1708959 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=89.4091796875, floorSegmentMB=1.666015625, forceMergeDeletesPctAllowed=29.841986508731416, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1708996 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.SolrIndexSearcher Opening [Searcher@7b452937[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] main]
[junit4] 2> 1709014 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1709015 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set
[junit4] 2> 1709015 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1709017 INFO (searcherExecutor-1844-thread-1-processing-n:127.0.0.1:48892_solr x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3 s:shard2 c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection r:core_node4) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SolrCore [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] Registered new searcher Searcher@7b452937[org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1709017 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589840254639013888
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.SyncStrategy http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/ has no replicas
[junit4] 2> 1709037 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1709056 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:48892/solr/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3/ shard2
[junit4] 2> 1709249 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709278 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709278 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709450 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1709452 INFO (qtp133797802-6501) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&newCollection=true&collection=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&version=2&replicaType=NRT&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&coreNodeName=core_node4&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3&action=CREATE&numShards=2&shard=shard2&wt=javabin} status=0 QTime=6135
[junit4] 2> 1709479 INFO (qtp67429917-6510) [n:127.0.0.1:35017_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1709590 INFO (zkCallback-1232-thread-2-processing-n:127.0.0.1:35017_solr) [n:127.0.0.1:35017_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709598 INFO (zkCallback-1230-thread-1-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1709598 INFO (zkCallback-1230-thread-2-processing-n:127.0.0.1:48892_solr) [n:127.0.0.1:48892_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection/state.json] for collection [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1710448 INFO (OverseerCollectionConfigSetProcessor-99365014872260614-127.0.0.1:35017_solr-n_0000000000) [n:127.0.0.1:35017_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1710498 INFO (qtp67429917-6510) [n:127.0.0.1:35017_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&property.config=solrconfig-tlog.xml&property.schema=schema_latest.xml&collection.configName=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_config-set&name=org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=10038
[junit4] 2> 1710498 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection failOnTimeout:true timeout (sec):330
[junit4] 1> -
[junit4] 1> replica:core_node2 rstate:active live:true
[junit4] 1> replica:core_node4 rstate:active live:true
[junit4] 1> no one is recoverying
[junit4] 2> 1710499 INFO (SUITE-TestCloudJSONFacetJoinDomain-seed#[12B4E9F85E054F40]-worker) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection
[junit4] 2> 1710535 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0 (1589840256206635008)]} 0 23
[junit4] 2> 1710560 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1589840256254869504)]} 0 1
[junit4] 2> 1710595 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1 (1589840256293666816)]} 0 0
[junit4] 2> 1710709 INFO (qtp133797802-6505) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2 (1589840256385941504)]} 0 26
[junit4] 2> 1710814 INFO (qtp133797802-6500) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[3 (1589840256444661760)]} 0 75
[junit4] 2> 1710816 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[4 (1589840256524353536)]} 0 0
[junit4] 2> 1710817 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[4 (1589840256525402112)]} 0 0
[junit4] 2> 1710818 INFO (qtp133797802-6505) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[5 (1589840256526450688)]} 0 0
[junit4] 2> 1710835 INFO (qtp133797802-6500) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[6 (1589840256544276480)]} 0 0
[junit4] 2> 1710836 INFO (qtp133797802-6505) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[7 (1589840256545325056)]} 0 0
[junit4] 2> 1710837 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[8 (1589840256546373632)]} 0 0
[junit4] 2> 1710837 INFO (qtp133797802-6500) [n:127.0.0.1:48892_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard2 r:core_node4 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard2_replica_n3] webapp=/solr path=/update params={wt=javabin&version=2}{add=[9 (1589840256547422208)]} 0 0
[junit4] 2> 1710838 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[10 (1589840256548470784)]} 0 0
[junit4] 2> 1710849 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[11 (1589840256560005120)]} 0 0
[junit4] 2> 1710850 INFO (qtp67429917-6512) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[12 (1589840256561053696)]} 0 0
[junit4] 2> 1710860 INFO (qtp67429917-6514) [n:127.0.0.1:35017_solr c:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection s:shard1 r:core_node2 x:org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] o.a.s.u.p.LogUpdateProcessorFactory [org.apache.solr.cloud.TestCloudJSONFacetJoinDomain_collection_shard1_replica_n1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[13 (1589840256570490880)]} 0 0
[j
[...truncated too long message...]
ve/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/bcprov-jdk15on-1.54.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/analysis/icu/lucene-analyzers-icu-7.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/contrib/analysis-extras/lib/icu4j-60.2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0264535385095341195582.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0264849570380174238636.suites -stdin
[junit4] ERROR: JVM J2 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262319053265880513310.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262637713925886892793.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:836: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:780: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1513: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1038: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262319053265880513310.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J2-20180117_112125_0262637713925886892793.syserr
Total time: 443 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #317 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 317 - Still unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/317/
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi
Error Message:
expected:<1> but was:<0>
Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
at __randomizedtesting.SeedInfo.seed([75C131060C852E87:22E8CAB3D777CC9C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.testReadApi(AutoScalingHandlerTest.java:725)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12659 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/init-core-data-001
[junit4] 2> 2193689 WARN (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=9 numCloses=9
[junit4] 2> 2193710 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2193711 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 2193712 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 2193712 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001
[junit4] 2> 2193712 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2193712 INFO (Thread-1839) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2193712 INFO (Thread-1839) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2193752 ERROR (Thread-1839) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2193819 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:35315
[junit4] 2> 2194046 INFO (zkConnectionManagerCallback-2441-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194170 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2194391 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@44a77e0{/solr,null,AVAILABLE}
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5b418e5a{SSL,[ssl, http/1.1]}{127.0.0.1:41864}
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.e.j.s.Server Started @2203742ms
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=41864}
[junit4] 2> 2194392 ERROR (jetty-launcher-2438-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2194392 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T21:23:03.571Z
[junit4] 2> 2194436 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3375abbd{/solr,null,AVAILABLE}
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5d3f40ce{SSL,[ssl, http/1.1]}{127.0.0.1:43663}
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.e.j.s.Server Started @2203847ms
[junit4] 2> 2194496 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=43663}
[junit4] 2> 2194497 ERROR (jetty-launcher-2438-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2194497 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T21:23:03.676Z
[junit4] 2> 2194544 INFO (zkConnectionManagerCallback-2443-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194544 INFO (zkConnectionManagerCallback-2445-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194553 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2194553 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2194743 INFO (jetty-launcher-2438-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35315/solr
[junit4] 2> 2194799 INFO (zkConnectionManagerCallback-2449-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194824 INFO (jetty-launcher-2438-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:35315/solr
[junit4] 2> 2194852 INFO (zkConnectionManagerCallback-2453-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194985 INFO (zkConnectionManagerCallback-2455-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2194997 INFO (zkConnectionManagerCallback-2457-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2195536 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2195537 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43663_solr
[junit4] 2> 2195537 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.Overseer Overseer (id=99361605739610118-127.0.0.1:43663_solr-n_0000000000) starting
[junit4] 2> 2195601 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43663_solr
[junit4] 2> 2195873 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2196041 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196078 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196091 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196092 INFO (jetty-launcher-2438-thread-2) [n:127.0.0.1:43663_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/.
[junit4] 2> 2196152 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2196153 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2196155 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41864_solr
[junit4] 2> 2196168 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2196274 INFO (zkCallback-2454-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2196522 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196574 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196574 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2196575 INFO (jetty-launcher-2438-thread-1) [n:127.0.0.1:41864_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/.
[junit4] 2> 2196732 INFO (zkConnectionManagerCallback-2463-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2196734 INFO (zkConnectionManagerCallback-2467-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2196735 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2196751 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:35315/solr ready
[junit4] 2> 2196859 INFO (TEST-AutoScalingHandlerTest.testDeleteUsedPolicy-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteUsedPolicy
[junit4] 2> 2197223 INFO (qtp1964390929-10760) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=2
[junit4] 2> 2197268 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 2197282 INFO (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=197
[junit4] 2> 2197284 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf&name=COLL1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2&policy=nodelete and sendToOCPQueue=true
[junit4] 2> 2197286 INFO (OverseerThreadFactory-3840-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Create collection COLL1
[junit4] 2> 2197346 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=3
[junit4] 2> 2197404 INFO (qtp2066369429-10765) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2197426 INFO (OverseerThreadFactory-3840-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 592118598 sessionWrapper.createTime 592118577050542, this.sessionWrapper.createTime 592118577050542
[junit4] 2> 2197586 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"COLL1",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"COLL1_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2197815 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=COLL1_shard1_replica_n1&action=CREATE&numShards=1&collection=COLL1&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2197815 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2197959 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2197983 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2198945 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2198989 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.IndexSchema [COLL1_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2198991 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2198991 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'COLL1_shard1_replica_n1' using configuration from collection COLL1, trusted=true
[junit4] 2> 2199008 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.COLL1.shard1.replica_n1' (registry 'solr.core.COLL1.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2199008 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2199008 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrCore [[COLL1_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/COLL1_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./COLL1_shard1_replica_n1/data/]
[junit4] 2> 2199257 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2199257 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2199258 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2199258 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2199272 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6ad16831[COLL1_shard1_replica_n1] main]
[junit4] 2> 2199304 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2199304 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2199304 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2199305 INFO (searcherExecutor-3845-thread-1-processing-n:127.0.0.1:43663_solr x:COLL1_shard1_replica_n1 s:shard1 c:COLL1 r:core_node2) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SolrCore [COLL1_shard1_replica_n1] Registered new searcher Searcher@6ad16831[COLL1_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2199305 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785697697398784
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43663/solr/COLL1_shard1_replica_n1/
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.SyncStrategy https://127.0.0.1:43663/solr/COLL1_shard1_replica_n1/ has no replicas
[junit4] 2> 2199381 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2199401 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:43663/solr/COLL1_shard1_replica_n1/ shard1
[junit4] 2> 2199575 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2199575 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2199611 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2199613 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:COLL1 s:shard1 r:core_node2 x:COLL1_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf&newCollection=true&name=COLL1_shard1_replica_n1&action=CREATE&numShards=1&collection=COLL1&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1798
[junit4] 2> 2199629 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2199733 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2199733 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/COLL1/state.json] for collection [COLL1] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2200629 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=conf&name=COLL1&nrtReplicas=1&action=CREATE&numShards=1&wt=javabin&version=2&policy=nodelete} status=0 QTime=3345
[junit4] 2> 2200633 ERROR (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.h.RequestHandlerBase org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, errors: [{remove-policy=nodelete, errorMessages=[policy : nodelete is being used by collection COLL1]}],
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.processOps(AutoScalingHandler.java:207)
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.handleRequestBody(AutoScalingHandler.java:145)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.api.ApiBag$ReqHandlerToApi.call(ApiBag.java:242)
[junit4] 2> at org.apache.solr.api.V2HttpCall.handleAdmin(V2HttpCall.java:311)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2200633 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=400 QTime=1
[junit4] 2> 2200634 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=COLL1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2200640 INFO (OverseerCollectionConfigSetProcessor-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2200641 INFO (OverseerThreadFactory-3840-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 2200641 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.COLL1.shard1.replica_n1, tag=null
[junit4] 2> 2200641 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1352bac3: rootName = solr_43663, domain = solr.core.COLL1.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.COLL1.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@625e23b9
[junit4] 2> 2200684 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.c.SolrCore [COLL1_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@7db03a62
[junit4] 2> 2200684 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.COLL1.shard1.replica_n1, tag=2108701282
[junit4] 2> 2200684 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.COLL1.shard1.leader, tag=2108701282
[junit4] 2> 2200707 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=COLL1_shard1_replica_n1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=65
[junit4] 2> 2201441 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=COLL1&action=DELETE&wt=javabin&version=2} status=0 QTime=806
[junit4] 2> 2201441 INFO (TEST-AutoScalingHandlerTest.testDeleteUsedPolicy-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testDeleteUsedPolicy
[junit4] 2> 2201465 INFO (TEST-AutoScalingHandlerTest.testErrorHandling-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testErrorHandling
[junit4] 2> 2201467 ERROR (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.h.RequestHandlerBase org.apache.solr.api.ApiBag$ExceptionWithErrObject: Error in command payload, errors: [{set-cluster-policy=[{cores=<10, node=#ANY}, {shard=#EACH, node=#ANY}, {nodeRole=overseer, replica=0}], errorMessages=[replica is required in {
[junit4] 2> "shard":"#EACH",
[junit4] 2> "node":"#ANY"}]}],
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.processOps(AutoScalingHandler.java:207)
[junit4] 2> at org.apache.solr.cloud.autoscaling.AutoScalingHandler.handleRequestBody(AutoScalingHandler.java:145)
[junit4] 2> at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:177)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdmin(HttpSolrCall.java:736)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:717)
[junit4] 2> at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:498)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:380)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:426)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2201468 INFO (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=400 QTime=2
[junit4] 2> 2201468 INFO (TEST-AutoScalingHandlerTest.testErrorHandling-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testErrorHandling
[junit4] 2> 2201496 INFO (TEST-AutoScalingHandlerTest.testReadApi-seed#[75C131060C852E87]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testReadApi
[junit4] 2> 2201514 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201528 INFO (qtp1964390929-10759) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=12
[junit4] 2> 2201580 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=68
[junit4] 2> 2201605 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 2201606 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 2201607 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=12
[junit4] 2> 2201616 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=1
[junit4] 2> 2201617 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201618 INFO (qtp1964390929-10757) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=10
[junit4] 2> 2201620 INFO (qtp2066369429-10765) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201633 INFO (qtp1964390929-10760) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=12
[junit4] 2> 2201637 INFO (qtp2066369429-10770) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=18
[junit4] 2> 2201639 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=1
[junit4] 2> 2201654 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201657 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201670 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling/diagnostics params={wt=javabin&version=2} status=0 QTime=29
[junit4] 2> 2201670 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=6&collection.configName=conf&maxShardsPerNode=10&name=readApiTestViolations&nrtReplicas=6&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2201676 INFO (OverseerThreadFactory-3840-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Create collection readApiTestViolations
[junit4] 2> 2201676 INFO (OverseerCollectionConfigSetProcessor-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2201677 ERROR (OverseerThreadFactory-3840-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerCollectionMessageHandler Collection: readApiTestViolations operation: create failed:org.apache.solr.common.SolrException: 'maxShardsPerNode>0' is not supported when autoScaling policies are used
[junit4] 2> at org.apache.solr.cloud.CreateCollectionCmd.buildReplicaPositions(CreateCollectionCmd.java:294)
[junit4] 2> at org.apache.solr.cloud.CreateCollectionCmd.call(CreateCollectionCmd.java:123)
[junit4] 2> at org.apache.solr.cloud.OverseerCollectionMessageHandler.processMessage(OverseerCollectionMessageHandler.java:243)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor$Runner.run(OverseerTaskProcessor.java:469)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 2201698 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=6&collection.configName=conf&maxShardsPerNode=10&name=readApiTestViolations&nrtReplicas=6&action=CREATE&numShards=1&wt=javabin&version=2} status=400 QTime=28
[junit4] 2> 2201701 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201710 INFO (qtp1964390929-10759) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201711 INFO (qtp2066369429-10772) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=11
[junit4] 2> 2201712 INFO (qtp2066369429-10765) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=6&collection.configName=conf&name=readApiTestViolations&nrtReplicas=6&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2201720 INFO (OverseerThreadFactory-3840-thread-4-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Create collection readApiTestViolations
[junit4] 2> 2201720 INFO (OverseerCollectionConfigSetProcessor-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2201721 WARN (OverseerThreadFactory-3840-thread-4-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.CreateCollectionCmd Specified number of replicas of 6 on collection readApiTestViolations is higher than the number of Solr instances currently live or live and part of your createNodeSet(2). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 2201726 INFO (qtp2066369429-10767) [n:127.0.0.1:43663_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201743 INFO (qtp1964390929-10755) [n:127.0.0.1:41864_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName,os.systemLoadAverage,memory.heap.usage&wt=javabin&version=2&group=solr.jvm,solr.node,solr.core} status=0 QTime=0
[junit4] 2> 2201766 INFO (OverseerThreadFactory-3840-thread-4-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 592122938 sessionWrapper.createTime 592122915211723, this.sessionWrapper.createTime 592122915211723
[junit4] 2> 2201995 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2201997 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41864/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202006 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202008 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n6",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41864/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202009 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n8",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:43663/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202027 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"readApiTestViolations",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"readApiTestViolations_shard1_replica_n10",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"https://127.0.0.1:41864/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2202228 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n1&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202230 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n2&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202230 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2202232 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node7&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n4&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202233 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node9&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n6&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202233 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node11&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n8&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202237 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node12&collection.configName=conf&newCollection=true&name=readApiTestViolations_shard1_replica_n10&action=CREATE&numShards=1&collection=readApiTestViolations&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2202370 INFO (zkCallback-2454-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2202370 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2202371 INFO (zkCallback-2454-thread-2-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2202386 INFO (zkCallback-2456-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2203376 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203392 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203410 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203449 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n10] Schema name=minimal
[junit4] 2> 2203449 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203451 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203451 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n10' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203465 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n4] Schema name=minimal
[junit4] 2> 2203467 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203467 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n4' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203469 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n6] Schema name=minimal
[junit4] 2> 2203470 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203471 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n6' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203472 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.core.readApiTestViolations.shard1.replica_n10' (registry 'solr.core.readApiTestViolations.shard1.replica_n10') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203472 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203472 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n10] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/readApiTestViolations_shard1_replica_n10], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/./readApiTestViolations_shard1_replica_n10/data/]
[junit4] 2> 2203477 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.readApiTestViolations.shard1.replica_n4' (registry 'solr.core.readApiTestViolations.shard1.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203477 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203477 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n4] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/readApiTestViolations_shard1_replica_n4], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./readApiTestViolations_shard1_replica_n4/data/]
[junit4] 2> 2203492 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.core.readApiTestViolations.shard1.replica_n6' (registry 'solr.core.readApiTestViolations.shard1.replica_n6') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203492 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203492 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n6] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/readApiTestViolations_shard1_replica_n6], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/./readApiTestViolations_shard1_replica_n6/data/]
[junit4] 2> 2203506 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203701 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2203749 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n2] Schema name=minimal
[junit4] 2> 2203750 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2203751 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203751 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n2' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203764 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_41864.solr.core.readApiTestViolations.shard1.replica_n2' (registry 'solr.core.readApiTestViolations.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203764 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203764 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n2] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/readApiTestViolations_shard1_replica_n2], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node1/./readApiTestViolations_shard1_replica_n2/data/]
[junit4] 2> 2203817 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.s.IndexSchema [readApiTestViolations_shard1_replica_n8] Schema name=minimal
[junit4] 2> 2203819 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203819 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n8' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203820 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2203820 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'readApiTestViolations_shard1_replica_n1' using configuration from collection readApiTestViolations, trusted=true
[junit4] 2> 2203832 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.readApiTestViolations.shard1.replica_n1' (registry 'solr.core.readApiTestViolations.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203832 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203832 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n1] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/readApiTestViolations_shard1_replica_n1], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./readApiTestViolations_shard1_replica_n1/data/]
[junit4] 2> 2203869 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_43663.solr.core.readApiTestViolations.shard1.replica_n8' (registry 'solr.core.readApiTestViolations.shard1.replica_n8') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@51a9e114
[junit4] 2> 2203869 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2203869 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrCore [[readApiTestViolations_shard1_replica_n8] ] Opening new SolrCore at [/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/readApiTestViolations_shard1_replica_n8], dataDir=[/home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001/tempDir-001/node2/./readApiTestViolations_shard1_replica_n8/data/]
[junit4] 2> 2204286 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204286 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204287 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204287 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204288 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@39c8a774[readApiTestViolations_shard1_replica_n4] main]
[junit4] 2> 2204343 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204359 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204371 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204371 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204371 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204372 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703010533376
[junit4] 2> 2204372 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204372 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204373 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204373 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204374 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204374 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204375 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=6 found=1 timeoutin=9999ms
[junit4] 2> 2204380 INFO (searcherExecutor-3851-thread-1-processing-n:127.0.0.1:43663_solr x:readApiTestViolations_shard1_replica_n4 s:shard1 c:readApiTestViolations r:core_node7) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n4] Registered new searcher Searcher@39c8a774[readApiTestViolations_shard1_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204380 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.s.SolrIndexSearcher Opening [Searcher@25b2c56c[readApiTestViolations_shard1_replica_n10] main]
[junit4] 2> 2204400 INFO (zkCallback-2456-thread-2-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204400 INFO (zkCallback-2456-thread-3-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204404 INFO (zkCallback-2454-thread-2-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204404 INFO (zkCallback-2454-thread-1-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [2])
[junit4] 2> 2204416 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204416 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204417 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204417 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204418 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204418 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204419 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204419 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204428 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.s.SolrIndexSearcher Opening [Searcher@31642988[readApiTestViolations_shard1_replica_n6] main]
[junit4] 2> 2204436 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@619a949e[readApiTestViolations_shard1_replica_n1] main]
[junit4] 2> 2204437 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.s.SolrIndexSearcher Opening [Searcher@14337e68[readApiTestViolations_shard1_replica_n8] main]
[junit4] 2> 2204444 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2204444 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2204445 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2204445 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2204446 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@622bb469[readApiTestViolations_shard1_replica_n2] main]
[junit4] 2> 2204453 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204453 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204453 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204453 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204454 INFO (qtp2066369429-10771) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703096516608
[junit4] 2> 2204456 INFO (searcherExecutor-3855-thread-1-processing-n:127.0.0.1:43663_solr x:readApiTestViolations_shard1_replica_n1 s:shard1 c:readApiTestViolations r:core_node3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n1] Registered new searcher Searcher@619a949e[readApiTestViolations_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204456 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204457 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204457 INFO (qtp2066369429-10769) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703099662336
[junit4] 2> 2204459 INFO (searcherExecutor-3854-thread-1-processing-n:127.0.0.1:43663_solr x:readApiTestViolations_shard1_replica_n8 s:shard1 c:readApiTestViolations r:core_node11) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n8] Registered new searcher Searcher@14337e68[readApiTestViolations_shard1_replica_n8] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204468 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204468 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204469 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204469 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204470 INFO (searcherExecutor-3850-thread-1-processing-n:127.0.0.1:41864_solr x:readApiTestViolations_shard1_replica_n10 s:shard1 c:readApiTestViolations r:core_node12) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n10] Registered new searcher Searcher@25b2c56c[readApiTestViolations_shard1_replica_n10] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204470 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2204470 INFO (qtp1964390929-10761) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node12 x:readApiTestViolations_shard1_replica_n10] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703113293824
[junit4] 2> 2204471 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204472 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204472 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2204606 INFO (searcherExecutor-3852-thread-1-processing-n:127.0.0.1:41864_solr x:readApiTestViolations_shard1_replica_n6 s:shard1 c:readApiTestViolations r:core_node9) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n6] Registered new searcher Searcher@31642988[readApiTestViolations_shard1_replica_n6] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204606 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2204607 INFO (qtp1964390929-10758) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703256948736
[junit4] 2> 2204609 INFO (qtp1964390929-10754) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node9 x:readApiTestViolations_shard1_replica_n6] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1589785703259045888
[junit4] 2> 2204620 INFO (searcherExecutor-3853-thread-1-processing-n:127.0.0.1:41864_solr x:readApiTestViolations_shard1_replica_n2 s:shard1 c:readApiTestViolations r:core_node5) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n2] Registered new searcher Searcher@622bb469[readApiTestViolations_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:43663/solr/readApiTestViolations_shard1_replica_n4/
[junit4] 2> 2204876 INFO (qtp2066369429-10766) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.u.PeerSync PeerSync: core=readApiTestViolations_shard1_replica_n4 url=https://127.0.0.1:43663/solr START replicas=[https://127.0.0.1:43663/solr/readApiTestViolations_shard1_replica_n1/, https://127.0.0.1:41864/solr/readApiTestViolations_shard1_replica_n2/, https://127.0.0.1:41864/solr/readApiTestViolations_shard1_replica_n6/, https://127.0.0.1:43663/solr/readApiTestViolations_shard1_replica_n8/, https://127.0.0.1:41864/solr/readApiTestViolations_shard
[...truncated too long message...]
ApiTestViolations] has occurred - updating... (live nodes size: [0])
[junit4] 2> 2213370 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.c.SolrCore [readApiTestViolations_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@23932760
[junit4] 2> 2213370 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.readApiTestViolations.shard1.replica_n2, tag=596846432
[junit4] 2> 2213370 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@15a5ebb3: rootName = solr_41864, domain = solr.core.readApiTestViolations.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.readApiTestViolations.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@36bb95a6
[junit4] 2> 2213370 INFO (zkCallback-2454-thread-3-processing-n:127.0.0.1:41864_solr) [n:127.0.0.1:41864_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [0])
[junit4] 2> 2213371 INFO (jetty-closer-2439-thread-3) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@27e49c02{/solr,null,UNAVAILABLE}
[junit4] 2> 2213380 INFO (zkCallback-2456-thread-1-processing-n:127.0.0.1:43663_solr) [n:127.0.0.1:43663_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/readApiTestViolations/state.json] for collection [readApiTestViolations] has occurred - updating... (live nodes size: [0])
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-2) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node3 x:readApiTestViolations_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=1860750354
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-1) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node7 x:readApiTestViolations_shard1_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=541773479
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.readApiTestViolations.shard1.replica_n8, tag=651912070
[junit4] 2> 2213406 INFO (coreCloseExecutor-3886-thread-3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@33aa9b5c: rootName = solr_43663, domain = solr.core.readApiTestViolations.shard1.replica_n8, service url = null, agent id = null] for registry solr.core.readApiTestViolations.shard1.replica_n8 / com.codahale.metrics.MetricRegistry@4cc67e32
[junit4] 2> 2213440 INFO (coreCloseExecutor-3888-thread-2) [n:127.0.0.1:41864_solr c:readApiTestViolations s:shard1 r:core_node5 x:readApiTestViolations_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=596846432
[junit4] 2> 2213474 INFO (coreCloseExecutor-3886-thread-3) [n:127.0.0.1:43663_solr c:readApiTestViolations s:shard1 r:core_node11 x:readApiTestViolations_shard1_replica_n8] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.readApiTestViolations.shard1.leader, tag=651912070
[junit4] 2> 2213535 INFO (jetty-closer-2439-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99361605739610118-127.0.0.1:43663_solr-n_0000000000) closing
[junit4] 2> 2213551 INFO (OverseerStateUpdate-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:43663_solr
[junit4] 2> 2213583 WARN (OverseerAutoScalingTriggerThread-99361605739610118-127.0.0.1:43663_solr-n_0000000000) [n:127.0.0.1:43663_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 2213676 INFO (jetty-closer-2439-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@44a77e0{/solr,null,UNAVAILABLE}
[junit4] 2> 2215171 INFO (jetty-closer-2439-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3375abbd{/solr,null,UNAVAILABLE}
[junit4] 2> 2215172 ERROR (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2215172 INFO (SUITE-AutoScalingHandlerTest-seed#[75C131060C852E87]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35315 35315
[junit4] 2> 2215483 INFO (Thread-1839) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:35315 35315
[junit4] 2> 2215483 WARN (Thread-1839) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2> 3 /solr/security.json
[junit4] 2> 3 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 18 /solr/collections/readApiTestViolations/state.json
[junit4] 2> 5 /solr/collections/COLL1/state.json
[junit4] 2> 4 /solr/clusterprops.json
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/live_nodes
[junit4] 2> 4 /solr/collections
[junit4] 2> 3 /solr/autoscaling/events/node_lost_trigger
[junit4] 2> 2 /solr/autoscaling/events/node_added_trigger
[junit4] 2> 2 /solr/autoscaling/events/node_added_trigger1
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoScalingHandlerTest_75C131060C852E87-001
[junit4] 2> Jan 16, 2018 9:23:24 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=747, maxMBSortInHeap=6.8184077987319505, sim=RandomSimilarity(queryNorm=true): {}, locale=fi, timezone=Asia/Shanghai
[junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=294681568,total=522190848
[junit4] 2> NOTE: All tests run in this JVM: [MetricsConfigTest, TestCloudRecovery, TestHalfAndHalfDocValues, TestJmxIntegration, TestNumericTerms64, DistributedVersionInfoTest, TestExecutePlanAction, TestManagedSynonymFilterFactory, HdfsBasicDistributedZk2Test, TestReloadAndDeleteDocs, TestOmitPositions, TestSubQueryTransformer, TestMultiWordSynonyms, TlogReplayBufferedWhileIndexingTest, TestQuerySenderNoQuery, BooleanFieldTest, ZkSolrClientTest, TestRestoreCore, ZkFailoverTest, TestExpandComponent, TestTolerantUpdateProcessorCloud, CollectionTooManyReplicasTest, TestStressInPlaceUpdates, TestHdfsBackupRestoreCore, ExitableDirectoryReaderTest, SuggestComponentContextFilterQueryTest, SimpleMLTQParserTest, ShardSplitTest, TestRandomCollapseQParserPlugin, TestBinaryResponseWriter, TestManagedResource, CircularListTest, ImplicitSnitchTest, TestManagedSchemaThreadSafety, TestSurroundQueryParser, TestSolrCoreSnapshots, TestLegacyNumericRangeQueryBuilder, ConvertedLegacyTest, TestCustomSort, WrapperMergePolicyFactoryTest, TestSizeLimitedDistributedMap, TestRTimerTree, PolyFieldTest, TestBulkSchemaAPI, TestApiFramework, TestSolrConfigHandlerCloud, OutputWriterTest, TestSolrCloudWithKerberosAlt, TestConfigSetsAPIExclusivity, PeerSyncWithIndexFingerprintCachingTest, ReplaceNodeTest, OverriddenZkACLAndCredentialsProvidersTest, ConjunctionSolrSpellCheckerTest, TestCustomStream, TestSolr4Spatial, SolrGraphiteReporterTest, DOMUtilTest, TestStressUserVersions, SolrMetricReporterTest, OpenCloseCoreStressTest, SolrIndexConfigTest, NumericFieldsTest, ShardRoutingCustomTest, TestLockTree, BufferStoreTest, SynonymTokenizerTest, TestIntervalFaceting, TestSimDistributedQueue, DirectUpdateHandlerTest, TestCloudJSONFacetJoinDomain, TestDefaultStatsCache, TestLuceneMatchVersion, AddSchemaFieldsUpdateProcessorFactoryTest, BigEndianAscendingWordSerializerTest, TestGeoJSONResponseWriter, LukeRequestHandlerTest, TestFastWriter, BasicAuthStandaloneTest, SampleTest, ConfigureRecoveryStrategyTest, TestInfoStreamLogging, TestFastLRUCache, TestHighFrequencyDictionaryFactory, SolrCmdDistributorTest, PreAnalyzedFieldTest, TestCollationField, ReplicaListTransformerTest, CdcrBidirectionalTest, TestAnalyzedSuggestions, TestInPlaceUpdatesStandalone, TestRangeQuery, TestDelegationWithHadoopAuth, TolerantUpdateProcessorTest, TestAuthenticationFramework, SuggesterTSTTest, TestSimpleTrackingShardHandler, HdfsUnloadDistributedZkTest, OpenExchangeRatesOrgProviderTest, AnalyticsMergeStrategyTest, TestDocTermOrds, AutoScalingHandlerTest]
[junit4] Completed [288/767 (1!)] on J1 in 22.34s, 8 tests, 1 failure <<< FAILURES!
[...truncated 49783 lines...]
[JENKINS] Lucene-Solr-Tests-7.x - Build # 316 - Failure
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/316/
9 tests failed.
FAILED: org.apache.solr.cloud.AddReplicaTest.test
Error Message:
core_node6:{"core":"addreplicatest_coll_shard1_replica_n5","base_url":"http://127.0.0.1:54964/solr","node_name":"127.0.0.1:54964_solr","state":"active","type":"NRT"}
Stack Trace:
java.lang.AssertionError: core_node6:{"core":"addreplicatest_coll_shard1_replica_n5","base_url":"http://127.0.0.1:54964/solr","node_name":"127.0.0.1:54964_solr","state":"active","type":"NRT"}
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:918A03DE6678FBE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.AddReplicaTest.test(AddReplicaTest.java:84)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Timeout on query awhollynewcollection_0
Stack Trace:
java.lang.AssertionError: Timeout on query awhollynewcollection_0
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:C939EB534EA8CDD3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:474)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.SystemLogListenerTest.test
Error Message:
wrong number of events added to .system expected:<9> but was:<10>
Stack Trace:
java.lang.AssertionError: wrong number of events added to .system expected:<9> but was:<10>
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:918A03DE6678FBE]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.apache.solr.cloud.autoscaling.SystemLogListenerTest.test(SystemLogListenerTest.java:166)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration
Error Message:
Path /autoscaling/nodeAdded/127.0.0.1:43124_solr wasn't created
Stack Trace:
java.lang.AssertionError: Path /autoscaling/nodeAdded/127.0.0.1:43124_solr wasn't created
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:99F617EB46AE2FA9]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testNodeMarkersRegistration(TriggerIntegrationTest.java:936)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode
Error Message:
did not finish processing changes
Stack Trace:
java.lang.AssertionError: did not finish processing changes
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:26A3824487D66D5E]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:297)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState
Error Message:
The trigger did not fire at all
Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:9711698725B03EB]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testNodeAddedTriggerRestoreState(TestTriggerIntegration.java:365)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventQueue
Error Message:
action wasn't interrupted
Stack Trace:
java.lang.AssertionError: action wasn't interrupted
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:48F9DD4941FC24B3]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testEventQueue(TestTriggerIntegration.java:638)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners
Error Message:
The trigger did not fire at all
Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:6C8C08F7F5F77F9F]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners(TestTriggerIntegration.java:953)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown
Error Message:
CapturedEvent{timestamp=16382616708492790, stage=IGNORED, actionName='null', event={ "id":"3a33e64329c1b4T630hdbkyc23ofe1af80y30t32", "source":"node_added_cooldown_trigger", "eventTime":16382612711522740, "eventType":"NODEADDED", "properties":{ "eventTimes":[16382612711522740], "cooldown":true, "nodeNames":["127.0.0.1:10019_solr"]}}, context={}, config={ "trigger":"node_added_cooldown_trigger", "afterAction":[], "stage":[ "FAILED", "SUCCEEDED", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener", "beforeAction":[]}, message='In cooldown period.'} expected:<SUCCEEDED> but was:<IGNORED>
Stack Trace:
java.lang.AssertionError: CapturedEvent{timestamp=16382616708492790, stage=IGNORED, actionName='null', event={
"id":"3a33e64329c1b4T630hdbkyc23ofe1af80y30t32",
"source":"node_added_cooldown_trigger",
"eventTime":16382612711522740,
"eventType":"NODEADDED",
"properties":{
"eventTimes":[16382612711522740],
"cooldown":true,
"nodeNames":["127.0.0.1:10019_solr"]}}, context={}, config={
"trigger":"node_added_cooldown_trigger",
"afterAction":[],
"stage":[
"FAILED",
"SUCCEEDED",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
"beforeAction":[]}, message='In cooldown period.'} expected:<SUCCEEDED> but was:<IGNORED>
at __randomizedtesting.SeedInfo.seed([814C9FE7489BE246:B0F2F203363197B4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown(TestTriggerIntegration.java:1107)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11847 lines...]
[junit4] Suite: org.apache.solr.cloud.AddReplicaTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/init-core-data-001
[junit4] 2> 745985 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 745996 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 745996 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 746012 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001
[junit4] 2> 746012 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 746044 INFO (Thread-248) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 746087 INFO (Thread-248) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 746158 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:44430
[junit4] 2> 746298 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
[junit4] 2> 746307 ERROR (Thread-248) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 747899 INFO (zkConnectionManagerCallback-133-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 747981 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748016 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748021 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748021 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 748069 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4afdb32{/solr,null,AVAILABLE}
[junit4] 2> 748089 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@24c7f7c4{HTTP/1.1,[http/1.1]}{127.0.0.1:59586}
[junit4] 2> 748089 INFO (jetty-launcher-130-thread-1) [ ] o.e.j.s.Server Started @780042ms
[junit4] 2> 748089 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59586}
[junit4] 2> 748090 ERROR (jetty-launcher-130-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748090 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:09.794Z
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4becc214{/solr,null,AVAILABLE}
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@796968e2{HTTP/1.1,[http/1.1]}{127.0.0.1:40739}
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.e.j.s.Server Started @780579ms
[junit4] 2> 748625 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40739}
[junit4] 2> 748626 ERROR (jetty-launcher-130-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748626 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:10.330Z
[junit4] 2> 748632 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b5454ca{/solr,null,AVAILABLE}
[junit4] 2> 748632 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1c42ff6{HTTP/1.1,[http/1.1]}{127.0.0.1:54964}
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.e.j.s.Server Started @780586ms
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54964}
[junit4] 2> 748633 ERROR (jetty-launcher-130-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748633 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:10.337Z
[junit4] 2> 748716 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3805658f{/solr,null,AVAILABLE}
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@677d9fd1{HTTP/1.1,[http/1.1]}{127.0.0.1:49516}
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.e.j.s.Server Started @780670ms
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=49516}
[junit4] 2> 748717 ERROR (jetty-launcher-130-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 748717 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 748718 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 748718 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 748718 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-01-16T01:04:10.422Z
[junit4] 2> 748720 INFO (zkConnectionManagerCallback-135-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 748767 INFO (zkConnectionManagerCallback-137-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 748776 INFO (zkConnectionManagerCallback-139-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 748785 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 748808 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 748869 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 749000 INFO (zkConnectionManagerCallback-141-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 749043 INFO (jetty-launcher-130-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 749121 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 749953 INFO (jetty-launcher-130-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 749954 INFO (jetty-launcher-130-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 750017 INFO (zkConnectionManagerCallback-146-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750237 INFO (jetty-launcher-130-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44430/solr
[junit4] 2> 750237 INFO (zkConnectionManagerCallback-151-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750413 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@1a0453cd
[junit4] 2> 750416 DEBUG (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@6fe5548f
[junit4] 2> 750452 INFO (zkConnectionManagerCallback-153-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750560 INFO (zkConnectionManagerCallback-157-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750649 DEBUG (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@68035e01
[junit4] 2> 750651 INFO (zkConnectionManagerCallback-161-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750651 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$39/1242430876@1078f5dc
[junit4] 2> 750688 INFO (zkConnectionManagerCallback-159-thread-1-processing-n:127.0.0.1:54964_solr) [n:127.0.0.1:54964_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750768 INFO (zkConnectionManagerCallback-165-thread-1-processing-n:127.0.0.1:40739_solr) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 750860 INFO (zkConnectionManagerCallback-163-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 752830 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 752832 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 752833 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 752835 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 752837 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 752839 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 752868 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 752873 DEBUG (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393545-127.0.0.1:59586_solr-n_0000000000
[junit4] 2> 752880 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59586_solr
[junit4] 2> 752982 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer Overseer (id=99356812771393545-127.0.0.1:59586_solr-n_0000000000) starting
[junit4] 2> 753012 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 753016 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753038 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 753039 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 753042 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 753080 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 753120 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 753129 DEBUG (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393547-127.0.0.1:49516_solr-n_0000000001
[junit4] 2> 753145 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99356812771393545-127.0.0.1:59586_solr-n_0000000000
[junit4] 2> 753160 DEBUG (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99356812771393545-127.0.0.1:59586_solr-n_0000000000 to know if I could be the leader
[junit4] 2> 753177 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49516_solr
[junit4] 2> 753200 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753204 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753301 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753301 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753347 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 753352 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 753435 INFO (zkCallback-160-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753435 INFO (zkCallback-162-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753473 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 753476 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753476 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753477 DEBUG (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:59586_solr as DOWN
[junit4] 2> 753480 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 753485 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 753492 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 753493 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 753508 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 753509 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 753512 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393548-127.0.0.1:40739_solr-n_0000000002
[junit4] 2> 753512 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753513 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 753521 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 753522 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99356812771393547-127.0.0.1:49516_solr-n_0000000001
[junit4] 2> 753536 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99356812771393547-127.0.0.1:49516_solr-n_0000000001 to know if I could be the leader
[junit4] 2> 753536 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 753537 DEBUG (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:40739_solr as DOWN
[junit4] 2> 753537 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 753537 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 753538 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 753538 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 753540 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue-work
[junit4] 2> 753540 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 753557 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59586_solr
[junit4] 2> 753596 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/security.json
[junit4] 2> 753609 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 753612 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 753621 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40739_solr
[junit4] 2> 753622 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 753640 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753641 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753641 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753641 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 753647 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:59586_solr"} current state version: 0
[junit4] 2> 753647 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:59586_solr
[junit4] 2> 753814 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753814 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753814 INFO (zkCallback-160-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 753814 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/autoscaling.json
[junit4] 2> 753815 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 753816 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753816 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 753816 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 753817 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:59586_solr, 127.0.0.1:40739_solr, 127.0.0.1:49516_solr]
[junit4] 2> 753817 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 753817 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 753817 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:40739_solr"} current state version: 0
[junit4] 2> 753818 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:40739_solr
[junit4] 2> 753846 INFO (zkCallback-162-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 753932 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 753952 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 754013 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 754014 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/99356812771393546-127.0.0.1:54964_solr-n_0000000003
[junit4] 2> 754032 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/overseer_elect/election/99356812771393548-127.0.0.1:40739_solr-n_0000000002
[junit4] 2> 754032 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/99356812771393548-127.0.0.1:40739_solr-n_0000000002 to know if I could be the leader
[junit4] 2> 754032 DEBUG (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.ZkController Publish node=127.0.0.1:54964_solr as DOWN
[junit4] 2> 754034 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 754035 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54964_solr
[junit4] 2> 754035 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754036 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754036 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754036 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/live_nodes
[junit4] 2> 754052 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754052 INFO (zkCallback-162-thread-1-processing-n:127.0.0.1:49516_solr) [n:127.0.0.1:49516_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 754077 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 754078 DEBUG (OverseerAutoScalingTriggerThread-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 754096 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 754116 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754136 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754137 INFO (zkCallback-160-thread-2-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 754212 INFO (zkCallback-158-thread-1-processing-n:127.0.0.1:54964_solr) [n:127.0.0.1:54964_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (4)
[junit4] 2> 754212 INFO (zkCallback-164-thread-1-processing-n:127.0.0.1:40739_solr) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (3)
[junit4] 2> 754097 DEBUG (ScheduledTrigger-172-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 754532 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 754533 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 754552 INFO (zkCallback-164-thread-2-processing-n:127.0.0.1:40739_solr) [n:127.0.0.1:40739_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 754564 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:54964_solr"} current state version: 0
[junit4] 2> 754564 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:54964_solr
[junit4] 2> 754791 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 754797 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 755556 DEBUG (ScheduledTrigger-172-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 755896 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 755897 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 758350 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 758352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 758572 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 758981 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54964.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 758984 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49516.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759345 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54964.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759346 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54964.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759377 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40739.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759421 INFO (jetty-launcher-130-thread-3) [n:127.0.0.1:54964_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node3/.
[junit4] 2> 759422 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49516.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759422 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_49516.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759596 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 759625 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59586.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 759628 INFO (jetty-launcher-130-thread-4) [n:127.0.0.1:49516_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node4/.
[junit4] 2> 760060 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40739.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760061 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40739.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760078 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59586.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760078 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_59586.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@2b13bf1f
[junit4] 2> 760170 INFO (jetty-launcher-130-thread-2) [n:127.0.0.1:40739_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node2/.
[junit4] 2> 760211 INFO (jetty-launcher-130-thread-1) [n:127.0.0.1:59586_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J1/temp/solr.cloud.AddReplicaTest_814C9FE7489BE246-001/tempDir-001/node1/.
[junit4] 2> 760352 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 760353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 760597 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 761598 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 762353 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 0 tasks from work-queue : [[]]
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 0
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: []
[junit4] 2> 762354 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 100 elements. ExcludeSet: []
[junit4] 2> 762482 INFO (zkConnectionManagerCallback-175-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 762572 INFO (zkConnectionManagerCallback-179-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 762573 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/clusterprops.json
[junit4] 2> 762582 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/live_nodes
[junit4] 2> 762582 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 762582 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: data: /solr/clusterstate.json
[junit4] 2> 762582 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 762592 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/aliases.json
[junit4] 2> 762593 INFO (SUITE-AddReplicaTest-seed#[814C9FE7489BE246]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:44430/solr ready
[junit4] 2> 762604 DEBUG (ScheduledTrigger-172-thread-2) [n:127.0.0.1:59586_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 762833 INFO (TEST-AddReplicaTest.test-seed#[814C9FE7489BE246]) [ ] o.a.s.SolrTestCaseJ4 ###Starting test
[junit4] 2> 762871 INFO (zkConnectionManagerCallback-181-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 762965 INFO (qtp1811337134-765) [n:127.0.0.1:49516_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=conf1&maxShardsPerNode=2&name=addreplicatest_coll&nrtReplicas=1&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 762968 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 762976 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: create/delete: /solr/overseer/collection-queue-work/qnr-0000000000
[junit4] 2> 762976 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 762976 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 762977 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/collection-queue-work
[junit4] 2> 762977 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/collection-queue-work
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.ZkDistributedQueue Found child node with improper name: qnr-0000000000
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Returning topN elements: [/overseer/collection-queue-work/qn-0000000000, ]
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Got 1 tasks from work-queue : [[org.apache.solr.cloud.OverseerTaskQueue$QueueEvent@2e684ecf]]
[junit4] 2> 762984 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Marked task [/overseer/collection-queue-work/qn-0000000000] as running
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
[junit4] 2> "name":"addreplicatest_coll",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Cleaning up work-queue. #Running tasks: 1
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor BlockedTasks: []
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor CompletedTasks: []
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor RunningZKTasks: [/overseer/collection-queue-work/qn-0000000000]
[junit4] 2> 762985 DEBUG (OverseerCollectionConfigSetProcessor-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskQueue Peeking for top 99 elements. ExcludeSet: [/overseer/collection-queue-work/qn-0000000000],[]
[junit4] 2> 763014 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerTaskProcessor Runner processing /overseer/collection-queue-work/qn-0000000000
[junit4] 2> 763014 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
[junit4] 2> "name":"addreplicatest_coll",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "collection.configName":"conf1",
[junit4] 2> "numShards":"2",
[junit4] 2> "maxShardsPerNode":"2",
[junit4] 2> "nrtReplicas":"1",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"}
[junit4] 2> 763014 INFO (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.CreateCollectionCmd Create collection addreplicatest_coll
[junit4] 2> 763024 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.Assign Identify nodes using default
[junit4] 2> 763024 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/addreplicatest_coll
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763033 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/collections
[junit4] 2> 763040 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763040 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763044 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.CreateCollectionCmd Check for collection zkNode:addreplicatest_coll
[junit4] 2> 763044 DEBUG (OverseerThreadFactory-174-thread-1-processing-n:127.0.0.1:59586_solr) [n:127.0.0.1:59586_solr ] o.a.s.c.CreateCollectionCmd Collection zkNode exists
[junit4] 2> 763052 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch fired: children: /solr/overseer/queue
[junit4] 2> 763054 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763054 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763054 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/collections
[junit4] 2> 763055 DEBUG (SyncThread:0) [ ] o.a.s.c.ZkTestServer Watch added: children: /solr/overseer/queue
[junit4] 2> 763056 DEBUG (OverseerStateUpdate-99356812771393545-127.0.0.1:59586_solr-n_0000000000) [n:127.0.0.1:59586_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "name":"addreplicatest_coll",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "collection.c
[...truncated too long message...]
ve/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptor-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-admin-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authn-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-authz-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-changelog-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/bcprov-jdk15on-1.54.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/byte-buddy-1.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/commons-math3-3.6.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-common-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-hdfs-2.7.4-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/hadoop-minikdc-2.7.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-sslengine-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/mockito-core-2.6.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/core/test-lib/objenesis-2.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/analysis/icu/lucene-analyzers-icu-7.3.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/build/backward-codecs/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/contrib/analysis-extras/lib/icu4j-60.2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.4.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_144/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.5.3.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5405703862273861884410.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_540569789209671531803.suites -stdin
[junit4] ERROR: JVM J1 ended with an exception: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5401070842046531995164.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5404755002079522906066.syserr
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1519)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:126)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:982)
[junit4] at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:979)
[junit4] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] at java.lang.Thread.run(Thread.java:748)
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:836: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:780: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build.xml:262: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/common-build.xml:556: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1513: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/lucene/common-build.xml:1038: At least one slave process threw an exception, first: Forked process returned with error code: 1. Very likely a JVM crash. See process stdout at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5401070842046531995164.sysout See process stderr at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/temp/junit4-J1-20180116_005109_5404755002079522906066.syserr
Total time: 395 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[Fast Archiver] No artifacts from Lucene-Solr-Tests-7.x #315 to compare, so performing full copy of artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-7.x - Build # 315 - Still Unstable
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-7.x/315/
5 tests failed.
FAILED: org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey
Error Message:
There are still nodes recoverying - waited for 330 seconds
Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
at __randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:45E424CF2DA2D387]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:185)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:140)
at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:135)
at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:913)
at org.apache.solr.cloud.ShardSplitTest.testSplitWithChaosMonkey(ShardSplitTest.java:428)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:993)
at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:968)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate
Error Message:
The trigger did not fire at all
Stack Trace:
java.lang.AssertionError: The trigger did not fire at all
at __randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:938BE997A362DE4C]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown
Error Message:
[CapturedEvent{timestamp=462370991008577, stage=SUCCEEDED, actionName='null', event={ "id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl", "source":"node_added_cooldown_trigger", "eventTime":462368175425427, "eventType":"NODEADDED", "properties":{ "eventTimes":[462368175425427], "_enqueue_time_":462369981194277, "nodeNames":["127.0.0.1:10005_solr"]}}, context={}, config={ "trigger":"node_added_cooldown_trigger", "afterAction":[], "stage":[ "FAILED", "SUCCEEDED", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener", "beforeAction":[]}, message='null'}]
Stack Trace:
java.lang.AssertionError: [CapturedEvent{timestamp=462370991008577, stage=SUCCEEDED, actionName='null', event={
"id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl",
"source":"node_added_cooldown_trigger",
"eventTime":462368175425427,
"eventType":"NODEADDED",
"properties":{
"eventTimes":[462368175425427],
"_enqueue_time_":462369981194277,
"nodeNames":["127.0.0.1:10005_solr"]}}, context={}, config={
"trigger":"node_added_cooldown_trigger",
"afterAction":[],
"stage":[
"FAILED",
"SUCCEEDED",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerListener",
"beforeAction":[]}, message='null'}]
at __randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:FF7D9AFA120E0DF1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testCooldown(TestTriggerIntegration.java:1100)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners
Error Message:
Stack Trace:
java.util.ConcurrentModificationException
at __randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:2303600ED1C8E5DA]:0)
at java.util.ArrayList$Itr.checkForComodification(ArrayList.java:909)
at java.util.ArrayList$Itr.next(ArrayList.java:859)
at java.util.AbstractCollection.toString(AbstractCollection.java:461)
at java.lang.String.valueOf(String.java:2994)
at java.lang.StringBuilder.append(StringBuilder.java:131)
at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testListeners(TestTriggerIntegration.java:962)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig
Error Message:
expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{ "cluster-preferences":[{"maximize":"freedisk"}], "triggers":{".auto_add_replicas":{ "name":".auto_add_replicas", "event":"nodeLost", "waitFor":30, "actions":[ { "name":"auto_add_replicas_plan", "class":"solr.AutoAddReplicasPlanAction"}, { "name":"execute_plan", "class":"solr.ExecutePlanAction"}], "enabled":true}}, "listeners":{".auto_add_replicas.system":{ "trigger":".auto_add_replicas", "afterAction":[], "stage":[ "STARTED", "ABORTED", "SUCCEEDED", "FAILED", "BEFORE_ACTION", "AFTER_ACTION", "IGNORED"], "class":"org.apache.solr.cloud.autoscaling.SystemLogListener", "beforeAction":[]}}, "properties":{}}>
Stack Trace:
java.lang.AssertionError: expected: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}> but was: org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig<{
"cluster-preferences":[{"maximize":"freedisk"}],
"triggers":{".auto_add_replicas":{
"name":".auto_add_replicas",
"event":"nodeLost",
"waitFor":30,
"actions":[
{
"name":"auto_add_replicas_plan",
"class":"solr.AutoAddReplicasPlanAction"},
{
"name":"execute_plan",
"class":"solr.ExecutePlanAction"}],
"enabled":true}},
"listeners":{".auto_add_replicas.system":{
"trigger":".auto_add_replicas",
"afterAction":[],
"stage":[
"STARTED",
"ABORTED",
"SUCCEEDED",
"FAILED",
"BEFORE_ACTION",
"AFTER_ACTION",
"IGNORED"],
"class":"org.apache.solr.cloud.autoscaling.SystemLogListener",
"beforeAction":[]}},
"properties":{}}>
at __randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:F14BF4B67BF288E4]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:147)
at org.apache.solr.cloud.autoscaling.sim.TestClusterStateProvider.testAutoScalingConfig(TestClusterStateProvider.java:214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12013 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestTriggerIntegration_CEC3F71E6CA47803-001/init-core-data-001
[junit4] 2> 568659 WARN (SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=2 numCloses=2
[junit4] 2> 568692 INFO (SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 568693 INFO (SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 568693 INFO (SUITE-TestTriggerIntegration-seed#[CEC3F71E6CA47803]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 568785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 568785 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
[junit4] 2> 568786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 568786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 568786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 568890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 568890 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
[junit4] 2> 568996 DEBUG (ScheduledTrigger-1073-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569078 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testEventQueue
[junit4] 2> 569078 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 569079 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 569080 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 569100 DEBUG (ScheduledTrigger-1073-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569151 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 569151 WARN (ScheduledTrigger-1073-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger ran but was already closed
[junit4] 2> 569151 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 569151 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 569152 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 569152 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569188 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569222 DEBUG (ScheduledTrigger-1076-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569262 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569282 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569371 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569408 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569415 DEBUG (simCloudManagerPool-1075-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 569415 DEBUG (simCloudManagerPool-1075-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 569416 DEBUG (simCloudManagerPool-1075-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 569434 INFO (simCloudManagerPool-1075-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
[junit4] 2> 569434 DEBUG (simCloudManagerPool-1075-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10001_solr, 127.0.0.1:10000_solr]
[junit4] 2> 569434 DEBUG (simCloudManagerPool-1075-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
[junit4] 2> 569434 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 569434 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
[junit4] 2> 569435 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 569444 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 569445 DEBUG (ScheduledTrigger-1076-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 569445 DEBUG (ScheduledTrigger-1076-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569445 DEBUG (ScheduledTrigger-1076-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10002_solr at time 462262255869127
[junit4] 2> 569469 DEBUG (ScheduledTrigger-1076-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:10002_solr] added at times [462262255869127], now=462263447164177
[junit4] 2> 569469 DEBUG (ScheduledTrigger-1076-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":462262255869127,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462262255869127],
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}}
[junit4] 2> 569470 DEBUG (AutoscalingActionExecutor-1077-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":462262255869127,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462262255869127],
[junit4] 2> "_enqueue_time_":462263461237677,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}}
[junit4] 2> 569471 INFO (AutoscalingActionExecutor-1077-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
[junit4] 2> "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":462262255869127,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462262255869127],
[junit4] 2> "_enqueue_time_":462263461237677,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"]}}
[junit4] 2> 569471 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 569471 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 569471 DEBUG (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 569472 WARN (AutoscalingActionExecutor-1077-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Exception executing actions
[junit4] 2> org.apache.lucene.store.AlreadyClosedException: ScheduledTrigger node_added_trigger1 has been closed.
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.dequeue(ScheduledTriggers.java:494)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.lambda$null$3(ScheduledTriggers.java:306)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 569472 DEBUG (AutoscalingActionExecutor-1077-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 70 ms for event id=1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 569472 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestEventQueueAction instantiated
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestEventQueueAction}], enabled=true}
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion -1
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Adding lost node from marker path: 127.0.0.1:10001_solr
[junit4] 2> 569472 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
[junit4] 2> 569473 DEBUG (ScheduledTrigger-1079-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569473 DEBUG (ScheduledTrigger-1079-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10001_solr
[junit4] 2> 569473 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 569473 DEBUG (ScheduledTrigger-1079-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":462262255869127,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462262255869127],
[junit4] 2> "_dequeue_time_":462263642306527,
[junit4] 2> "_enqueue_time_":462263461237677,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"],
[junit4] 2> "replaying":true}}
[junit4] 2> 569473 DEBUG (ScheduledTrigger-1079-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 569473 DEBUG (ScheduledTrigger-1079-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 569474 DEBUG (AutoscalingActionExecutor-1080-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":462262255869127,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462262255869127],
[junit4] 2> "_dequeue_time_":462263642306527,
[junit4] 2> "_enqueue_time_":462263461237677,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"],
[junit4] 2> "replaying":true}}
[junit4] 2> 569474 INFO (AutoscalingActionExecutor-1080-thread-1) [ ] o.a.s.c.a.s.TestTriggerIntegration -- event: {
[junit4] 2> "id":"1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":462262255869127,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462262255869127],
[junit4] 2> "_dequeue_time_":462263642306527,
[junit4] 2> "_enqueue_time_":462263461237677,
[junit4] 2> "nodeNames":["127.0.0.1:10002_solr"],
[junit4] 2> "replaying":true}}
[junit4] 2> 569582 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testEventQueue
[junit4] 2> 569582 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase
[junit4] 2>
[junit4] 2> 569582 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2> 569582 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
[junit4] 2> 569582 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2>
[junit4] 2> 569582 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2
[junit4] 2> 569582 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 2
[junit4] 2> 569583 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 1
[junit4] 2> 569583 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10001_solr
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: []
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 1
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2
[junit4] 2> 569584 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
[junit4] 2> 569585 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger1
[junit4] 2> 569585 INFO (TEST-TestTriggerIntegration.testEventQueue-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 2
[junit4] 2> 569595 DEBUG (AutoscalingActionExecutor-1080-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 6091 ms for event id=1a46cd18ddcc7Tdh05m3uxqd3e1aovccmuk6ld7
[junit4] 2> 569601 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTrigger
[junit4] 2> 569601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 569601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimNodeStateProvider - removing dead node values: 127.0.0.1:10001_solr
[junit4] 2> 569601 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 569601 DEBUG (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 569619 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
[junit4] 2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion -1
[junit4] 2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 569620 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
[junit4] 2> 569631 DEBUG (ScheduledTrigger-1082-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569655 DEBUG (ScheduledTrigger-1082-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569704 DEBUG (ScheduledTrigger-1082-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
[junit4] 2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 569721 DEBUG (simCloudManagerPool-1081-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 569722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
[junit4] 2> 569722 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
[junit4] 2> 569722 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
[junit4] 2> 569763 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 569763 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569763 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10003_solr at time 462278160743427
[junit4] 2> 569764 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:10003_solr] added at times [462278160743427], now=462279638884277
[junit4] 2> 569793 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"1a470858efc03Tdh05m3uxqd3e1aovccmuk6ldb",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":462278160743427,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462278160743427],
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 569794 DEBUG (AutoscalingActionExecutor-1083-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"1a470858efc03Tdh05m3uxqd3e1aovccmuk6ldb",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":462278160743427,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462278160743427],
[junit4] 2> "_enqueue_time_":462279646129827,
[junit4] 2> "nodeNames":["127.0.0.1:10003_solr"]}}
[junit4] 2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
[junit4] 2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
[junit4] 2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr, 127.0.0.1:10003_solr]
[junit4] 2> 569795 DEBUG (simCloudManagerPool-1081-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 569795 DEBUG (AutoscalingActionExecutor-1083-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 78 ms for event id=1a470858efc03Tdh05m3uxqd3e1aovccmuk6ldb
[junit4] 2> 569815 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
[junit4] 2> 569815 DEBUG (ScheduledTrigger-1082-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 569815 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
[junit4] 2> 569829 DEBUG (ScheduledTrigger-1082-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 569829 DEBUG (ScheduledTrigger-1082-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569835 DEBUG (ScheduledTrigger-1082-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTrigger
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase
[junit4] 2>
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2>
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 3
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 3
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0
[junit4] 2> 569835 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: []
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 2
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_trigger
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1
[junit4] 2> 569836 INFO (TEST-TestTriggerIntegration.testNodeAddedTrigger-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 1
[junit4] 2> 569851 DEBUG (ScheduledTrigger-1082-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 569851 DEBUG (ScheduledTrigger-1082-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569890 DEBUG (ScheduledTrigger-1082-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 569890 DEBUG (ScheduledTrigger-1082-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 569890 DEBUG (ScheduledTrigger-1082-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569919 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 569919 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 569919 DEBUG (ScheduledTrigger-1082-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 569920 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testSearchRate
[junit4] 2> 569920 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
[junit4] 2> 569921 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 569921 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 569921 DEBUG (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
[junit4] 2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion -1
[junit4] 2> 569923 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
[junit4] 2> 569943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 569943 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
[junit4] 2> 570012 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570032 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570057 DEBUG (ScheduledTrigger-1085-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570093 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570121 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570149 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570178 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570195 DEBUG (simCloudManagerPool-1084-thread-4) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 570195 DEBUG (simCloudManagerPool-1084-thread-4) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
[junit4] 2> 570232 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570252 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570267 DEBUG (simCloudManagerPool-1084-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 570267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
[junit4] 2> 570267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 570267 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
[junit4] 2> 570287 DEBUG (ScheduledTrigger-1085-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570323 DEBUG (ScheduledTrigger-1085-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570340 DEBUG (simCloudManagerPool-1084-thread-5) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 570340 DEBUG (simCloudManagerPool-1084-thread-5) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 11
[junit4] 2> 570401 DEBUG (ScheduledTrigger-1085-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570403 DEBUG (simCloudManagerPool-1084-thread-5) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 570403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 11
[junit4] 2> 570403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 570403 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 11, lastZnodeVersion 11
[junit4] 2> 570421 DEBUG (ScheduledTrigger-1085-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570485 DEBUG (ScheduledTrigger-1085-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570506 DEBUG (ScheduledTrigger-1085-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570619 DEBUG (ScheduledTrigger-1085-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570647 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570684 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570685 DEBUG (ScheduledTrigger-1085-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"1a478f37a9251Tdh05m3uxqd3e1aovccmuk6lde",
[junit4] 2> "source":"search_rate_trigger",
[junit4] 2> "eventTime":462314364637777,
[junit4] 2> "eventType":"SEARCHRATE",
[junit4] 2> "properties":{
[junit4] 2> "node":{
[junit4] 2> "127.0.0.1:10000_solr":250.0,
[junit4] 2> "127.0.0.1:10002_solr":250.0},
[junit4] 2> "replica":[
[junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10000_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}",
[junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10002_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"],
[junit4] 2> "collection":{"collection1":500.0},
[junit4] 2> "shard":{"collection1":{"shard1":500.0}}}}
[junit4] 2> 570801 DEBUG (AutoscalingActionExecutor-1086-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"1a478f37a9251Tdh05m3uxqd3e1aovccmuk6lde",
[junit4] 2> "source":"search_rate_trigger",
[junit4] 2> "eventTime":462314364637777,
[junit4] 2> "eventType":"SEARCHRATE",
[junit4] 2> "properties":{
[junit4] 2> "node":{
[junit4] 2> "127.0.0.1:10000_solr":250.0,
[junit4] 2> "127.0.0.1:10002_solr":250.0},
[junit4] 2> "replica":[
[junit4] 2> "{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10000_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}",
[junit4] 2> "{\"core_node1\":{\n \"core\":\"collection1_shard1_replica_n1\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10002_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"],
[junit4] 2> "collection":{"collection1":500.0},
[junit4] 2> "shard":{"collection1":{"shard1":500.0}},
[junit4] 2> "_enqueue_time_":462324241524177}}
[junit4] 2> 570856 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testSearchRate
[junit4] 2> 570856 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase
[junit4] 2>
[junit4] 2> 570856 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2> 570856 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
[junit4] 2> 570856 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2>
[junit4] 2> 570856 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 2
[junit4] 2> 570857 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 0
[junit4] 2> 570857 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0
[junit4] 2> 570857 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [collection1]
[junit4] 2> 570857 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 1
[junit4] 2> 570857 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 1
[junit4] 2> 570857 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 2
[junit4] 2> 570857 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * collection1 2
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - active 2
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - CREATE 1
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 1
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: search_rate_trigger
[junit4] 2> 570858 INFO (TEST-TestTriggerIntegration.testSearchRate-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 1
[junit4] 2> 570948 DEBUG (AutoscalingActionExecutor-1086-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 12198 ms for event id=1a478f37a9251Tdh05m3uxqd3e1aovccmuk6lde
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestTriggerIntegration -Dtests.method=testSearchRate -Dtests.seed=CEC3F71E6CA47803 -Dtests.multiplier=2 -Dtests.slow=true -Dtests.locale=ja-JP -Dtests.timezone=Europe/Vaduz -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] FAILURE 1.11s J2 | TestTriggerIntegration.testSearchRate <<<
[junit4] > Throwable #1: java.lang.AssertionError: The trigger did not fire at all
[junit4] > at __randomizedtesting.SeedInfo.seed([CEC3F71E6CA47803:938BE997A362DE4C]:0)
[junit4] > at org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration.testSearchRate(TestTriggerIntegration.java:1178)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 570984 DEBUG (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 570987 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCooldown
[junit4] 2> 570987 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 12
[junit4] 2> 570987 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 12
[junit4] 2> 570996 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 570996 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 570996 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 12, lastZnodeVersion 12
[junit4] 2> 571012 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 571012 WARN (ScheduledTrigger-1085-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Ignoring autoscaling event {
[junit4] 2> "id":"1a47aa54d14a3Tdh05m3uxqd3e1aovccmuk6ldh",
[junit4] 2> "source":"search_rate_trigger",
[junit4] 2> "eventTime":462321642968227,
[junit4] 2> "eventType":"SEARCHRATE",
[junit4] 2> "properties":{
[junit4] 2> "node":{"127.0.0.1:10000_solr":250.0},
[junit4] 2> "replica":["{\"core_node2\":{\n \"core\":\"collection1_shard1_replica_n2\",\n \"INDEX.sizeInBytes\":123450000,\n \"rate\":250.0,\n \"node_name\":\"127.0.0.1:10000_solr\",\n \"state\":\"active\",\n \"type\":\"NRT\",\n \"QUERY./select.requestTimes:1minRate\":250.0,\n \"shard\":\"shard1\",\n \"collection\":\"collection1\"}}"],
[junit4] 2> "collection":{"collection1":250.0},
[junit4] 2> "shard":{"collection1":{"shard1":250.0}}}} because the source trigger: search_rate_trigger doesn't exist.
[junit4] 2> 571107 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 571121 DEBUG (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 13
[junit4] 2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion -1
[junit4] 2> 571142 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 13
[junit4] 2> 571162 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 571162 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 13, lastZnodeVersion 13
[junit4] 2> 571162 DEBUG (ScheduledTrigger-1088-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 571328 DEBUG (ScheduledTrigger-1088-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 571331 DEBUG (simCloudManagerPool-1087-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 571331 DEBUG (simCloudManagerPool-1087-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 14
[junit4] 2> 571348 DEBUG (ScheduledTrigger-1088-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 571361 DEBUG (simCloudManagerPool-1087-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 571361 DEBUG (simCloudManagerPool-1087-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 571361 DEBUG (simCloudManagerPool-1087-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 571361 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 14
[junit4] 2> 571361 INFO (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.s.TestTriggerIntegration TestTriggerAction init
[junit4] 2> 571361 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 14, lastZnodeVersion 14
[junit4] 2> 571361 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 571361 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 15
[junit4] 2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.sim.TestTriggerIntegration$TestTriggerAction}], enabled=true}
[junit4] 2> 571362 DEBUG (simCloudManagerPool-1087-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr, 127.0.0.1:10002_solr]
[junit4] 2> 571363 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 15
[junit4] 2> 571363 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 15, lastZnodeVersion 15
[junit4] 2> 571368 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 571382 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 571382 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 571382 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10004_solr at time 462359080776977
[junit4] 2> 571389 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10004_solr] added at times [462359080776977], now=462360854030827
[junit4] 2> 571417 DEBUG (ScheduledTrigger-1088-thread-3) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"1a4835cc4b511Tdh05m3uxqd3e1aovccmuk6ldi",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":462359080776977,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462359080776977],
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}}
[junit4] 2> 571418 DEBUG (AutoscalingActionExecutor-1089-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"1a4835cc4b511Tdh05m3uxqd3e1aovccmuk6ldi",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":462359080776977,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462359080776977],
[junit4] 2> "_enqueue_time_":462360861686077,
[junit4] 2> "nodeNames":["127.0.0.1:10004_solr"]}}
[junit4] 2> 571418 DEBUG (AutoscalingActionExecutor-1089-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 17 ms for event id=1a4835cc4b511Tdh05m3uxqd3e1aovccmuk6ldi
[junit4] 2> 571439 DEBUG (ScheduledTrigger-1088-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 571563 DEBUG (ScheduledTrigger-1088-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 3
[junit4] 2> 571564 DEBUG (ScheduledTrigger-1088-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 571564 DEBUG (ScheduledTrigger-1088-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 571564 DEBUG (ScheduledTrigger-1088-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:10005_solr at time 462368175425427
[junit4] 2> 571599 DEBUG (ScheduledTrigger-1088-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_cooldown_trigger firing registered processor for nodes: [127.0.0.1:10005_solr] added at times [462368175425427], now=462369974245877
[junit4] 2> 571600 DEBUG (ScheduledTrigger-1088-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":462368175425427,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462368175425427],
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}
[junit4] 2> 571617 DEBUG (AutoscalingActionExecutor-1089-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl",
[junit4] 2> "source":"node_added_cooldown_trigger",
[junit4] 2> "eventTime":462368175425427,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[462368175425427],
[junit4] 2> "_enqueue_time_":462369981194277,
[junit4] 2> "nodeNames":["127.0.0.1:10005_solr"]}}
[junit4] 2> 571620 DEBUG (AutoscalingActionExecutor-1089-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 147 ms for event id=1a4857ada0793Tdh05m3uxqd3e1aovccmuk6ldl
[junit4] 2> 571636 DEBUG (ScheduledTrigger-1088-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 571637 DEBUG (ScheduledTrigger-1088-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 571637 DEBUG (ScheduledTrigger-1088-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 571672 DEBUG (ScheduledTrigger-1088-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 4
[junit4] 2> 571672 DEBUG (ScheduledTrigger-1088-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_cooldown_trigger
[junit4] 2> 571672 DEBUG (ScheduledTrigger-1088-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 571672 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testCooldown
[junit4] 2> 571672 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase
[junit4] 2>
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2>
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 4
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 4
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 0
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: []
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 0
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 0
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 0
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 2
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 4
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegration.testCooldown-seed#[CEC3F71E6CA47803]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_added_cooldown_trigger
[junit4] 2> 571673 INFO (TEST-TestTriggerIntegratio
[...truncated too long message...]
] 2> 3577646 INFO (zkCallback-7765-thread-3-processing-n:127.0.0.1:42976_solr) [n:127.0.0.1:42976_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 3580143 WARN (zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) [n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SyncStrategy Closed, skipping sync up.
[junit4] 2> 3580143 INFO (zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) [n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 3580143 INFO (zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) [n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.c.SolrCore [.system_shard1_replica_n2] CLOSING SolrCore org.apache.solr.core.SolrCore@54e775ec
[junit4] 2> 3580144 INFO (zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) [n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core..system.shard1.replica_n2, tag=1424455148
[junit4] 2> 3580144 INFO (zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) [n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@25b32ffe: rootName = solr_42976, domain = solr.core..system.shard1.replica_n2, service url = null, agent id = null] for registry solr.core..system.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@6aad0356
[junit4] 2> 3580151 INFO (zkCallback-7765-thread-1-processing-n:127.0.0.1:42976_solr) [n:127.0.0.1:42976_solr c:.system s:shard1 r:core_node5 x:.system_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection..system.shard1.leader, tag=1424455148
[junit4] 2> 3580152 INFO (jetty-closer-7747-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@1acc79ee{/solr,null,UNAVAILABLE}
[junit4] 2> 3580153 ERROR (SUITE-TestClusterStateProvider-seed#[CEC3F71E6CA47803]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3580153 INFO (SUITE-TestClusterStateProvider-seed#[CEC3F71E6CA47803]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38210 38210
[junit4] 2> 3580210 INFO (Thread-8872) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:38210 38210
[junit4] 2> 3580211 WARN (Thread-8872) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 3 /solr/configs/.system
[junit4] 2> 3 /solr/configs/.system/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 26 /solr/collections/.system/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 3 /solr/autoscaling.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-7.x/solr/build/solr-core/test/J2/temp/solr.cloud.autoscaling.sim.TestClusterStateProvider_CEC3F71E6CA47803-001
[junit4] 2> Jan 15, 2018 10:08:24 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=367, maxMBSortInHeap=7.668266411331945, sim=RandomSimilarity(queryNorm=true): {}, locale=sr-Latn-RS, timezone=America/Argentina/La_Rioja
[junit4] 2> NOTE: Linux 4.4.0-104-generic amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=4,threads=1,free=109349360,total=510656512
[junit4] 2> NOTE: All tests run in this JVM: [TestSolrCloudWithSecureImpersonation, TestFieldCacheSortRandom, TestCSVResponseWriter, TestSolrQueryResponse, SuggesterWFSTTest, TestSolrCoreProperties, TestFieldCache, TestEmbeddedSolrServerConstructors, LeaderElectionTest, BadComponentTest, SolrMetricsIntegrationTest, TestCollectionsAPIViaSolrCloudCluster, HdfsThreadLeakTest, NodeLostTriggerTest, IndexSchemaTest, ClassificationUpdateProcessorIntegrationTest, SolrCoreCheckLockOnStartupTest, TestNestedDocsSort, TestPullReplica, TestRealTimeGet, CoreAdminRequestStatusTest, TestDistribIDF, TestTolerantUpdateProcessorRandomCloud, TestQuerySenderListener, TestTriggerIntegration, HttpTriggerListenerTest, TestHashQParserPlugin, TestPartialUpdateDeduplication, TestSolrFieldCacheBean, ShufflingReplicaListTransformerTest, BlockJoinFacetSimpleTest, DisMaxRequestHandlerTest, DocValuesNotIndexedTest, TestCloudSchemaless, RequestLoggingTest, TestNamedUpdateProcessors, TestReloadDeadlock, TestHttpShardHandlerFactory, SharedFSAutoReplicaFailoverTest, SolrTestCaseJ4Test, QueryElevationComponentTest, TestFileDictionaryLookup, TestFieldCollectionResource, TestLeaderInitiatedRecoveryThread, TestMaxTokenLenTokenizer, TestTrieFacet, TestCloudSearcherWarming, CoreSorterTest, TestSolrConfigHandlerConcurrent, TestSystemIdResolver, PKIAuthenticationIntegrationTest, HdfsBasicDistributedZkTest, TestGroupingSearch, RecoveryAfterSoftCommitTest, EchoParamsTest, ZkNodePropsTest, TestUniqueKeyFieldResource, TestGraphMLResponseWriter, SpellCheckCollatorWithCollapseTest, TestTrie, CdcrBootstrapTest, TestSolrCLIRunExample, TestFunctionQuery, TriLevelCompositeIdRoutingTest, FieldMutatingUpdateProcessorTest, HdfsDirectoryFactoryTest, TestConfigSetsAPIZkFailure, TestPolicyCloud, StatsComponentTest, CheckHdfsIndexTest, TestNoOpRegenerator, LargeFieldTest, DistributedFacetPivotSmallAdvancedTest, TestCoreDiscovery, DocExpirationUpdateProcessorFactoryTest, TestGenericDistributedQueue, DistributedFacetExistsSmallTest, TestSolrDeletionPolicy2, SuggestComponentTest, DeleteNodeTest, TestLegacyFieldCache, TestMinMaxOnMultiValuedField, HighlighterTest, TestLRUCache, BlockCacheTest, RequiredFieldsTest, TestJsonRequest, TestCopyFieldCollectionResource, TestMiniSolrCloudClusterSSL, TestShardHandlerFactory, TestStressLiveNodes, TestCoreContainer, TimeZoneUtilsTest, TestHashPartitioner, TestComplexPhraseLeadingWildcard, LoggingHandlerTest, TestHdfsUpdateLog, SortByFunctionTest, CoreAdminHandlerTest, DeleteStatusTest, TestExtendedDismaxParser, TestSegmentSorting, SSLMigrationTest, DistributedSpellCheckComponentTest, OverseerStatusTest, LeaderElectionContextKeyTest, MetricsHandlerTest, TestFiltering, TestSchemaSimilarityResource, TestFuzzyAnalyzedSuggestions, ConfigSetsAPITest, SolrJmxReporterCloudTest, DefaultValueUpdateProcessorTest, TestNumericRangeQuery32, TestTolerantSearch, OverseerCollectionConfigSetProcessorTest, AutoAddReplicasIntegrationTest, ChaosMonkeyNothingIsSafeTest, SubstringBytesRefFilterTest, TestRuleBasedAuthorizationPlugin, LeaderInitiatedRecoveryOnCommitTest, PreAnalyzedFieldManagedSchemaCloudTest, AtomicUpdateProcessorFactoryTest, TestDynamicLoading, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, BasicZkTest, RecoveryZkTest, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, TestDistributedSearch, DistributedTermsComponentTest, TestSolr4Spatial, PeerSyncTest, DirectUpdateHandlerTest, TestIndexSearcher, SoftAutoCommitTest, HighlighterConfigTest, UpdateParamsTest, AlternateDirectoryTest, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, TestMergePolicyConfig, MultiTermTest, TestDocSet, TestBinaryField, TestSearchPerf, NumericFieldsTest, TestElisionMultitermQuery, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastWriter, ResourceLoaderTest, TestFastOutputStream, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, DateMathParserTest, PreAnalyzedFieldTest, PrimUtilsTest, TestSuggestSpellingConverter, RAMDirectoryFactoryTest, ClusterStateTest, TestSolrJ, TestUtils, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, CircularListTest, DistributedMLTComponentTest, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestEmbeddedSolrServerAdminHandler, TestEmbeddedSolrServerSchemaAPI, ConnectionReuseTest, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, ChaosMonkeySafeLeaderWithPullReplicasTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, DeleteInactiveReplicaTest, DistribJoinFromCollectionTest, ForceLeaderTest, MoveReplicaHDFSTest, TestCloudDeleteByQuery, TestCloudPseudoReturnFields, TestClusterProperties, TestCollectionAPI, TestExclusionRuleCollectionAccess, TestLeaderElectionZkExpiry, TestLocalFSCloudBackupRestore, TestRandomRequestDistribution, TestSolrCloudWithDelegationTokens, TestSolrCloudWithKerberosAlt, TestTlogReplica, TestClusterStateProvider]
[junit4] Completed [586/767 (3!)] on J2 in 12.74s, 2 tests, 1 failure <<< FAILURES!
[...truncated 48844 lines...]