You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2018/02/19 15:17:06 UTC
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1687 - Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1687/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication
Error Message:
Index: 0, Size: 0
Stack Trace:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
at __randomizedtesting.SeedInfo.seed([9F16DB95344B09D5:8B5E80C0174CB4CB]:0)
at java.util.ArrayList.rangeCheck(ArrayList.java:657)
at java.util.ArrayList.get(ArrayList.java:433)
at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandler: 1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandler:
1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([9F16DB95344B09D5]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([9F16DB95344B09D5]:0)
Build Log:
[...truncated 13313 lines...]
[junit4] Suite: org.apache.solr.handler.TestReplicationHandler
[junit4] 2> 2519220 INFO (SUITE-TestReplicationHandler-seed#[9F16DB95344B09D5]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/init-core-data-001
[junit4] 2> 2519220 WARN (SUITE-TestReplicationHandler-seed#[9F16DB95344B09D5]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=10 numCloses=10
[junit4] 2> 2519220 INFO (SUITE-TestReplicationHandler-seed#[9F16DB95344B09D5]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2519221 INFO (SUITE-TestReplicationHandler-seed#[9F16DB95344B09D5]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 2519223 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.SolrTestCaseJ4 ###Starting doTestIndexAndConfigAliasReplication
[junit4] 2> 2519224 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/collection1
[junit4] 2> 2519226 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2519227 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2519227 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2519227 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 2519227 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@592d037d{/solr,null,AVAILABLE}
[junit4] 2> 2519227 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@acb1d52{HTTP/1.1,[http/1.1]}{127.0.0.1:43051}
[junit4] 2> 2519227 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.Server Started @2525374ms
[junit4] 2> 2519228 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=43051}
[junit4] 2> 2519228 ERROR (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2519228 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2519228 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 2519228 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2519228 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-02-19T14:50:08.608Z
[junit4] 2> 2519228 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/solr.xml
[junit4] 2> 2519232 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2519232 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2519233 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2519265 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519272 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519272 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519274 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/.
[junit4] 2> 2519274 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2519274 INFO (coreLoadExecutor-7027-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2519289 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2519294 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2519295 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 2519295 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/./collection1, trusted=true
[junit4] 2> 2519295 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519295 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2519295 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/./collection1/data/]
[junit4] 2> 2519297 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=317440432, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2519333 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2519333 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2519333 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=38.79296875, floorSegmentMB=2.0107421875, forceMergeDeletesPctAllowed=6.114332710433047, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2519333 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@61810201[collection1] main]
[junit4] 2> 2519334 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-001/collection1/conf
[junit4] 2> 2519334 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
[junit4] 2> 2519334 INFO (coreLoadExecutor-7027-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2519335 INFO (searcherExecutor-7028-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@61810201[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2519336 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/collection1
[junit4] 2> 2519338 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-21T15:27:37-06:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 2519340 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 2519340 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 2519340 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 2519340 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@34590386{/solr,null,AVAILABLE}
[junit4] 2> 2519340 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@73183fce{HTTP/1.1,[http/1.1]}{127.0.0.1:45682}
[junit4] 2> 2519340 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.e.j.s.Server Started @2525487ms
[junit4] 2> 2519340 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=45682}
[junit4] 2> 2519341 ERROR (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2519341 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 2519341 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 2519341 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2519341 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-02-19T14:50:08.721Z
[junit4] 2> 2519341 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/solr.xml
[junit4] 2> 2519344 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2519344 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2519344 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 2519375 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519389 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519389 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519391 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/.
[junit4] 2> 2519391 INFO (TEST-TestReplicationHandler.doTestIndexAndConfigAliasReplication-seed#[9F16DB95344B09D5]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 2519392 INFO (coreLoadExecutor-7037-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 2519401 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 2519405 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 2519406 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 2519407 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/./collection1, trusted=true
[junit4] 2> 2519407 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@7659162d
[junit4] 2> 2519407 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2519407 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/./collection1/data/]
[junit4] 2> 2519409 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=317440432, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 2519448 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2519448 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2519448 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=38.79296875, floorSegmentMB=2.0107421875, forceMergeDeletesPctAllowed=6.114332710433047, segmentsPerTier=25.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 2519449 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1d3e2e45[collection1] main]
[junit4] 2> 2519449 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J1/temp/solr.handler.TestReplicationHandler_9F16DB95344B09D5-001/solr-instance-002/collection1/conf
[junit4] 2> 2519450 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
[junit4] 2> 2519451 INFO (coreLoadExecutor-7037-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2519451 INFO (searcherExecutor-7038-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@1d3e2e45[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2519453 INFO (qtp557722466-22342) [ x:collection1] o.a.s.c.S.Request [collection1] webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 2519456 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 0
[junit4] 2> 2519457 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 0
[junit4] 2> 2519457 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
[junit4] 2> 2519458 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 0
[junit4] 2> 2519458 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[4]} 0 0
[junit4] 2> 2519459 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[5]} 0 0
[junit4] 2> 2519459 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[6]} 0 0
[junit4] 2> 2519459 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[7]} 0 0
[junit4] 2> 2519460 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[8]} 0 0
[junit4] 2> 2519460 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[9]} 0 0
[junit4] 2> 2519461 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[10]} 0 0
[junit4] 2> 2519461 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[11]} 0 0
[junit4] 2> 2519461 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[12]} 0 0
[junit4] 2> 2519462 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[13]} 0 0
[junit4] 2> 2519462 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[14]} 0 0
[junit4] 2> 2519462 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[15]} 0 0
[junit4] 2> 2519463 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[16]} 0 0
[junit4] 2> 2519463 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[17]} 0 0
[junit4] 2> 2519463 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[18]} 0 0
[junit4] 2> 2519464 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[19]} 0 0
[junit4] 2> 2519464 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[20]} 0 0
[junit4] 2> 2519464 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[21]} 0 0
[junit4] 2> 2519465 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[22]} 0 0
[junit4] 2> 2519465 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[23]} 0 0
[junit4] 2> 2519466 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[24]} 0 0
[junit4] 2> 2519466 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[25]} 0 0
[junit4] 2> 2519466 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[26]} 0 0
[junit4] 2> 2519467 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[27]} 0 0
[junit4] 2> 2519467 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[28]} 0 0
[junit4] 2> 2519467 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[29]} 0 0
[junit4] 2> 2519468 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[30]} 0 0
[junit4] 2> 2519468 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[31]} 0 0
[junit4] 2> 2519468 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[32]} 0 0
[junit4] 2> 2519469 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[33]} 0 0
[junit4] 2> 2519469 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[34]} 0 0
[junit4] 2> 2519469 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[35]} 0 0
[junit4] 2> 2519470 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[36]} 0 0
[junit4] 2> 2519470 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[37]} 0 0
[junit4] 2> 2519470 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[38]} 0 0
[junit4] 2> 2519470 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[39]} 0 0
[junit4] 2> 2519471 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[40]} 0 0
[junit4] 2> 2519471 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[41]} 0 0
[junit4] 2> 2519472 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[42]} 0 0
[junit4] 2> 2519472 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[43]} 0 0
[junit4] 2> 2519472 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[44]} 0 0
[junit4] 2> 2519473 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[45]} 0 0
[junit4] 2> 2519473 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[46]} 0 0
[junit4] 2> 2519473 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[47]} 0 0
[junit4] 2> 2519474 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[48]} 0 0
[junit4] 2> 2519474 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[49]} 0 0
[junit4] 2> 2519474 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[50]} 0 0
[junit4] 2> 2519475 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[51]} 0 0
[junit4] 2> 2519475 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[52]} 0 0
[junit4] 2> 2519476 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[53]} 0 0
[junit4] 2> 2519476 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[54]} 0 0
[junit4] 2> 2519476 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[55]} 0 0
[junit4] 2> 2519477 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[56]} 0 0
[junit4] 2> 2519477 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[57]} 0 0
[junit4] 2> 2519477 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[58]} 0 0
[junit4] 2> 2519478 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[59]} 0 0
[junit4] 2> 2519491 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[60]} 0 0
[junit4] 2> 2519491 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[61]} 0 0
[junit4] 2> 2519491 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[62]} 0 0
[junit4] 2> 2519492 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[63]} 0 0
[junit4] 2> 2519492 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[64]} 0 0
[junit4] 2> 2519492 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[65]} 0 0
[junit4] 2> 2519493 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[66]} 0 0
[junit4] 2> 2519493 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[67]} 0 0
[junit4] 2> 2519493 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[68]} 0 0
[junit4] 2> 2519494 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[69]} 0 0
[junit4] 2> 2519494 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[70]} 0 0
[junit4] 2> 2519494 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[71]} 0 0
[junit4] 2> 2519495 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[72]} 0 0
[junit4] 2> 2519495 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[73]} 0 0
[junit4] 2> 2519496 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[74]} 0 0
[junit4] 2> 2519496 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[75]} 0 0
[junit4] 2> 2519496 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[76]} 0 0
[junit4] 2> 2519497 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[77]} 0 0
[junit4] 2> 2519497 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[78]} 0 0
[junit4] 2> 2519497 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[79]} 0 0
[junit4] 2> 2519498 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[80]} 0 0
[junit4] 2> 2519498 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[81]} 0 0
[junit4] 2> 2519498 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[82]} 0 0
[junit4] 2> 2519499 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[83]} 0 0
[junit4] 2> 2519499 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[84]} 0 0
[junit4] 2> 2519499 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[85]} 0 0
[junit4] 2> 2519500 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[86]} 0 0
[junit4] 2> 2519500 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[87]} 0 0
[junit4] 2> 2519500 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[88]} 0 0
[junit4] 2> 2519501 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[89]} 0 0
[junit4] 2> 2519501 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[90]} 0 0
[junit4] 2> 2519501 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[91]} 0 0
[junit4] 2> 2519502 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[92]} 0 0
[junit4] 2> 2519502 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[93]} 0 0
[junit4] 2> 2519502 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[94]} 0 0
[junit4] 2> 2519503 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[95]} 0 0
[junit4] 2> 2519503 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[96]} 0 0
[junit4] 2> 2519503 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[97]} 0 0
[junit4] 2> 2519504 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[98]} 0 0
[junit4] 2> 2519504 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[99]} 0 0
[junit4] 2> 2519504 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[100]} 0 0
[junit4] 2> 2519505 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[101]} 0 0
[junit4] 2> 2519505 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[102]} 0 0
[junit4] 2> 2519505 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[103]} 0 0
[junit4] 2> 2519505 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[104]} 0 0
[junit4] 2> 2519506 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[105]} 0 0
[junit4] 2> 2519506 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[106]} 0 0
[junit4] 2> 2519506 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[107]} 0 0
[junit4] 2> 2519506 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[108]} 0 0
[junit4] 2> 2519507 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[109]} 0 0
[junit4] 2> 2519507 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[110]} 0 0
[junit4] 2> 2519508 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[111]} 0 0
[junit4] 2> 2519508 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[112]} 0 0
[junit4] 2> 2519508 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[113]} 0 0
[junit4] 2> 2519508 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[114]} 0 0
[junit4] 2> 2519509 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[115]} 0 0
[junit4] 2> 2519509 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[116]} 0 0
[junit4] 2> 2519510 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[117]} 0 0
[junit4] 2> 2519510 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[118]} 0 0
[junit4] 2> 2519510 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[119]} 0 0
[junit4] 2> 2519511 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[120]} 0 0
[junit4] 2> 2519511 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[121]} 0 0
[junit4] 2> 2519511 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[122]} 0 0
[junit4] 2> 2519512 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[123]} 0 0
[junit4] 2> 2519512 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[124]} 0 0
[junit4] 2> 2519512 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[125]} 0 0
[junit4] 2> 2519512 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[126]} 0 0
[junit4] 2> 2519513 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[127]} 0 0
[junit4] 2> 2519513 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[128]} 0 0
[junit4] 2> 2519513 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[129]} 0 0
[junit4] 2> 2519513 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[130]} 0 0
[junit4] 2> 2519514 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[131]} 0 0
[junit4] 2> 2519514 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[132]} 0 0
[junit4] 2> 2519514 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[133]} 0 0
[junit4] 2> 2519514 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[134]} 0 0
[junit4] 2> 2519514 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[135]} 0 0
[junit4] 2> 2519515 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[136]} 0 0
[junit4] 2> 2519515 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[137]} 0 0
[junit4] 2> 2519515 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[138]} 0 0
[junit4] 2> 2519515 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[139]} 0 0
[junit4] 2> 2519516 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[140]} 0 0
[junit4] 2> 2519516 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[141]} 0 0
[junit4] 2> 2519516 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[142]} 0 0
[junit4] 2> 2519516 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[143]} 0 0
[junit4] 2> 2519517 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[144]} 0 0
[junit4] 2> 2519517 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[145]} 0 0
[junit4] 2> 2519517 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[146]} 0 0
[junit4] 2> 2519517 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[147]} 0 0
[junit4] 2> 2519518 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[148]} 0 0
[junit4] 2> 2519518 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[149]} 0 0
[junit4] 2> 2519518 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[150]} 0 0
[junit4] 2> 2519518 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[151]} 0 0
[junit4] 2> 2519519 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[152]} 0 0
[junit4] 2> 2519519 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[153]} 0 0
[junit4] 2> 2519519 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[154]} 0 0
[junit4] 2> 2519519 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[155]} 0 0
[junit4] 2> 2519520 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[156]} 0 0
[junit4] 2> 2519520 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[157]} 0 0
[junit4] 2> 2519520 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[158]} 0 0
[junit4] 2> 2519521 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[159]} 0 0
[junit4] 2> 2519521 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[160]} 0 0
[junit4] 2> 2519521 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[161]} 0 0
[junit4] 2> 2519521 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[162]} 0 0
[junit4] 2> 2519521 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[163]} 0 0
[junit4] 2> 2519522 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[164]} 0 0
[junit4] 2> 2519522 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[165]} 0 0
[junit4] 2> 2519522 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[166]} 0 0
[junit4] 2> 2519522 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[167]} 0 0
[junit4] 2> 2519523 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[168]} 0 0
[junit4] 2> 2519523 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[169]} 0 0
[junit4] 2> 2519523 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[170]} 0 0
[junit4] 2> 2519523 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[171]} 0 0
[junit4] 2> 2519524 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[172]} 0 0
[junit4] 2> 2519524 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[173]} 0 0
[junit4] 2> 2519524 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[174]} 0 0
[junit4] 2> 2519525 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[175]} 0 0
[junit4] 2> 2519525 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[176]} 0 0
[junit4] 2> 2519525 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[177]} 0 0
[junit4] 2> 2519525 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[178]} 0 0
[junit4] 2> 2519526 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[179]} 0 0
[junit4] 2> 2519526 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[180]} 0 0
[junit4] 2> 2519526 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[181]} 0 0
[junit4] 2> 2519526 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[182]} 0 0
[junit4] 2> 2519527 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[183]} 0 0
[junit4] 2> 2519527 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[184]} 0 0
[junit4] 2> 2519527 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[185]} 0 0
[junit4] 2> 2519528 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[186]} 0 0
[junit4] 2> 2519528 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[187]} 0 0
[junit4] 2> 2519528 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[188]} 0 0
[junit4] 2> 2519529 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[189]} 0 0
[junit4] 2> 2519529 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[190]} 0 0
[junit4] 2> 2519529 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[191]} 0 0
[junit4] 2> 2519529 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[192]} 0 0
[junit4] 2> 2519530 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[193]} 0 0
[junit4] 2> 2519530 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[194]} 0 0
[junit4] 2> 2519531 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[195]} 0 0
[junit4] 2> 2519531 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[196]} 0 0
[junit4] 2> 2519531 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[197]} 0 0
[junit4] 2> 2519532 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[198]} 0 0
[junit4] 2> 2519532 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[199]} 0 0
[junit4] 2> 2519532 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[200]} 0 0
[junit4] 2> 2519533 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[201]} 0 0
[junit4] 2> 2519533 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[202]} 0 0
[junit4] 2> 2519533 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[203]} 0 0
[junit4] 2> 2519534 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[204]} 0 0
[junit4] 2> 2519534 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[205]} 0 0
[junit4] 2> 2519534 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[206]} 0 0
[junit4] 2> 2519535 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[207]} 0 0
[junit4] 2> 2519535 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[208]} 0 0
[junit4] 2> 2519535 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[209]} 0 0
[junit4] 2> 2519535 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[210]} 0 0
[junit4] 2> 2519536 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[211]} 0 0
[junit4] 2> 2519536 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[212]} 0 0
[junit4] 2> 2519536 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[213]} 0 0
[junit4] 2> 2519537 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[214]} 0 0
[junit4] 2> 2519537 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[215]} 0 0
[junit4] 2> 2519537 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[216]} 0 0
[junit4] 2> 2519538 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[217]} 0 0
[junit4] 2> 2519538 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[218]} 0 0
[junit4] 2> 2519538 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[219]} 0 0
[junit4] 2> 2519538 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[220]} 0 0
[junit4] 2> 2519539 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[221]} 0 0
[junit4] 2> 2519539 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[222]} 0 0
[junit4] 2> 2519539 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[223]} 0 0
[junit4] 2> 2519539 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[224]} 0 0
[junit4] 2> 2519540 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[225]} 0 0
[junit4] 2> 2519540 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[226]} 0 0
[junit4] 2> 2519540 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[227]} 0 0
[junit4] 2> 2519541 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[228]} 0 0
[junit4] 2> 2519541 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[229]} 0 0
[junit4] 2> 2519541 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[230]} 0 0
[junit4] 2> 2519541 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[231]} 0 0
[junit4] 2> 2519542 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[232]} 0 0
[junit4] 2> 2519542 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[233]} 0 0
[junit4] 2> 2519542 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[234]} 0 0
[junit4] 2> 2519543 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[235]} 0 0
[junit4] 2> 2519543 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[236]} 0 0
[junit4] 2> 2519543 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[237]} 0 0
[junit4] 2> 2519543 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[238]} 0 0
[junit4] 2> 2519544 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[239]} 0 0
[junit4] 2> 2519544 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[240]} 0 0
[junit4] 2> 2519544 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[241]} 0 0
[junit4] 2> 2519544 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[242]} 0 0
[junit4] 2> 2519545 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[243]} 0 0
[junit4] 2> 2519545 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[244]} 0 0
[junit4] 2> 2519545 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[245]} 0 0
[junit4] 2> 2519545 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[246]} 0 0
[junit4] 2> 2519546 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[247]} 0 0
[junit4] 2> 2519546 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[248]} 0 0
[junit4] 2> 2519546 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[249]} 0 0
[junit4] 2> 2519547 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[250]} 0 0
[junit4] 2> 2519547 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[251]} 0 0
[junit4] 2> 2519547 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[252]} 0 0
[junit4] 2> 2519547 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[253]} 0 0
[junit4] 2> 2519548 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[254]} 0 0
[junit4] 2> 2519548 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[255]} 0 0
[junit4] 2> 2519548 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[256]} 0 0
[junit4] 2> 2519548 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[257]} 0 0
[junit4] 2> 2519548 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[258]} 0 0
[junit4] 2> 2519549 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[259]} 0 0
[junit4] 2> 2519549 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[260]} 0 0
[junit4] 2> 2519549 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[261]} 0 0
[junit4] 2> 2519549 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[262]} 0 0
[junit4] 2> 2519549 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[263]} 0 0
[junit4] 2> 2519550 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[264]} 0 0
[junit4] 2> 2519550 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[265]} 0 0
[junit4] 2> 2519550 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[266]} 0 0
[junit4] 2> 2519550 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[267]} 0 0
[junit4] 2> 2519551 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[268]} 0 0
[junit4] 2> 2519551 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[269]} 0 0
[junit4] 2> 2519551 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[270]} 0 0
[junit4] 2> 2519552 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[271]} 0 0
[junit4] 2> 2519552 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[272]} 0 0
[junit4] 2> 2519552 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[273]} 0 0
[junit4] 2> 2519552 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[274]} 0 0
[junit4] 2> 2519553 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[275]} 0 0
[junit4] 2> 2519553 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[276]} 0 0
[junit4] 2> 2519553 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[277]} 0 0
[junit4] 2> 2519554 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[278]} 0 0
[junit4] 2> 2519554 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[279]} 0 0
[junit4] 2> 2519554 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[280]} 0 0
[junit4] 2> 2519554 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[281]} 0 0
[junit4] 2> 2519555 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[282]} 0 0
[junit4] 2> 2519555 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[283]} 0 0
[junit4] 2> 2519555 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[284]} 0 0
[junit4] 2> 2519555 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[285]} 0 0
[junit4] 2> 2519556 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[286]} 0 0
[junit4] 2> 2519556 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[287]} 0 0
[junit4] 2> 2519556 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[288]} 0 0
[junit4] 2> 2519556 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[289]} 0 0
[junit4] 2> 2519557 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[290]} 0 0
[junit4] 2> 2519557 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[291]} 0 0
[junit4] 2> 2519557 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[292]} 0 0
[junit4] 2> 2519558 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[293]} 0 0
[junit4] 2> 2519558 INFO (qtp557722466-22342) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[294]} 0 0
[junit4] 2> 2519558 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[295]} 0 0
[junit4] 2> 2519559 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[296]} 0 0
[junit4] 2> 2519559 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[297]} 0 0
[junit4] 2> 2519559 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[298]} 0 0
[junit4] 2> 2519559 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[299]} 0 0
[junit4] 2> 2519560 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[300]} 0 0
[junit4] 2> 2519560 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[301]} 0 0
[junit4] 2> 2519560 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[302]} 0 0
[junit4] 2> 2519561 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[303]} 0 0
[junit4] 2> 2519561 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[304]} 0 0
[junit4] 2> 2519561 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[305]} 0 0
[junit4] 2> 2519561 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[306]} 0 0
[junit4] 2> 2519562 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[307]} 0 0
[junit4] 2> 2519562 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[308]} 0 0
[junit4] 2> 2519562 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[309]} 0 0
[junit4] 2> 2519562 INFO (qtp557722466-22345) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[310]} 0 0
[junit4] 2> 2519563 INFO (qtp557722466-22348) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[311]} 0 0
[junit4]
[...truncated too long message...]
ervedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Feb 19, 2018 2:52:19 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
[junit4] 2> Feb 19, 2018 2:52:22 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Lucene70, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@5e74e433), locale=vi, timezone=Cuba
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=2,free=166458056,total=364892160
[junit4] 2> NOTE: All tests run in this JVM: [AsyncCallRequestStatusResponseTest, InfixSuggestersTest, SolrXmlInZkTest, TestPHPSerializedResponseWriter, ClusterStateUpdateTest, TestSearchPerf, TestNodeAddedTrigger, TestComponentsName, TestMacros, TestSolrIndexConfig, TestPolicyCloud, BigEndianAscendingWordDeserializerTest, TestSchemaResource, TestRawResponseWriter, TestFieldCacheWithThreads, HighlighterConfigTest, ExternalFileFieldSortTest, TestNumericTerms32, TestCrossCoreJoin, OutOfBoxZkACLAndCredentialsProvidersTest, TestLockTree, TestRandomFaceting, TestExclusionRuleCollectionAccess, OverseerTaskQueueTest, TestUtilizeNode, SmileWriterTest, PreAnalyzedUpdateProcessorTest, ConjunctionSolrSpellCheckerTest, BasicAuthStandaloneTest, ReplicationFactorTest, TestCloudNestedDocsSort, FullHLLTest, TestHighFrequencyDictionaryFactory, BlockJoinFacetSimpleTest, PeerSyncTest, TestRebalanceLeaders, TestExceedMaxTermLength, TestDistribStateManager, BadCopyFieldTest, DistanceFunctionTest, PreAnalyzedFieldTest, TestSortByMinMaxFunction, NotRequiredUniqueKeyTest, AddSchemaFieldsUpdateProcessorFactoryTest, UpdateLogTest, HealthCheckHandlerTest, BlockJoinFacetDistribTest, MetricUtilsTest, RequestHandlersTest, ConcurrentCreateRoutedAliasTest, TemplateUpdateProcessorTest, TestRandomDVFaceting, TestHashQParserPlugin, BadIndexSchemaTest, TestGenericDistributedQueue, TestDeleteCollectionOnDownNodes, TestRTGBase, DebugComponentTest, TestStressReorder, TestPerFieldSimilarityWithDefaultOverride, AutoAddReplicasIntegrationTest, TestScoreJoinQPNoScore, SimpleMLTQParserTest, DOMUtilTest, URLClassifyProcessorTest, CdcrReplicationDistributedZkTest, LeaderElectionContextKeyTest, TestStreamBody, ActionThrottleTest, TriLevelCompositeIdRoutingTest, TestSolrCloudWithHadoopAuthPlugin, SimpleFacetsTest, SolrMetricManagerTest, TestIndexingPerformance, TestFieldResource, TestDistributedGrouping, TestInPlaceUpdatesDistrib, TestManagedSchemaAPI, TestObjectReleaseTracker, TestFieldCacheReopen, TestDFRSimilarityFactory, TestSolrConfigHandler, HdfsDirectoryFactoryTest, ChaosMonkeyNothingIsSafeTest, CreateRoutedAliasTest, LargeFieldTest, TestSolr4Spatial, HdfsRecoverLeaseTest, CdcrRequestHandlerTest, TestGeoJSONResponseWriter, TestSolrCoreParser, TestHashPartitioner, TestPolicyCloud, ChaosMonkeySafeLeaderWithPullReplicasTest, QueryParsingTest, TestNRTOpen, TestShardHandlerFactory, TestHdfsCloudBackupRestore, TestUninvertingReader, TestCopyFieldCollectionResource, HDFSCollectionsAPITest, FullSolrCloudDistribCmdsTest, XsltUpdateRequestHandlerTest, RecoveryAfterSoftCommitTest, HighlighterMaxOffsetTest, CleanupOldIndexTest, SystemInfoHandlerTest, TestRandomFlRTGCloud, TestEmbeddedSolrServerConstructors, TestSubQueryTransformerCrossCore, TestInfoStreamLogging, TestSearcherReuse, DocValuesMultiTest, TestPayloadCheckQParserPlugin, CoreAdminHandlerTest, TestCloudPseudoReturnFields, CollectionStateFormat2Test, ShufflingReplicaListTransformerTest, DeleteStatusTest, TimeZoneUtilsTest, ChangedSchemaMergeTest, TestPullReplica, UUIDUpdateProcessorFallbackTest, SliceStateTest, TestTrackingShardHandlerFactory, DocValuesNotIndexedTest, StressHdfsTest, LeaderFailureAfterFreshStartTest, TestFilteredDocIdSet, TestTriggerIntegration, DisMaxRequestHandlerTest, SuggestComponentContextFilterQueryTest, BadComponentTest, OverseerRolesTest, DistributedSuggestComponentTest, TestCoreDiscovery, ClassificationUpdateProcessorIntegrationTest, TestSchemalessBufferedUpdates, TestNoOpRegenerator, SaslZkACLProviderTest, DistributedFacetPivotSmallTest, WrapperMergePolicyFactoryTest, RequestLoggingTest, FileBasedSpellCheckerTest, TestTolerantUpdateProcessorCloud, TestSmileRequest, DirectoryFactoryTest, DataDrivenBlockJoinTest, ExecutePlanActionTest, TestDynamicFieldResource, AutoScalingHandlerTest, TestOnReconnectListenerSupport, SpellCheckCollatorTest, BitVectorTest, DistributedQueryComponentCustomSortTest, TestRetrieveFieldsOptimizer, BasicDistributedZk2Test, DateRangeFieldTest, TestStressRecovery, DeleteReplicaTest, TestStressLiveNodes, LoggingHandlerTest, TestLRUCache, RegexBoostProcessorTest, FieldAnalysisRequestHandlerTest, TestLegacyField, TestMultiValuedNumericRangeQuery, TestPointFields, HdfsThreadLeakTest, TestAuthorizationFramework, HttpPartitionTest, ResponseLogComponentTest, TestSkipOverseerOperations, TestJettySolrRunner, TestXmlQParserPlugin, NumberUtilsTest, BlockDirectoryTest, TestJavabinTupleStreamParser, TestStressVersions, DeleteLastCustomShardedReplicaTest, TestDocTermOrds, TestAnalyzeInfixSuggestions, TestAnalyzedSuggestions, TestDistributedMissingSort, TestChildDocTransformer, HdfsRestartWhileUpdatingTest, SuggesterWFSTTest, ZkSolrClientTest, UtilsToolTest, TestLeaderElectionZkExpiry, CSVRequestHandlerTest, ChaosMonkeySafeLeaderTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, TestReplicationHandler]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestReplicationHandler -Dtests.seed=9F16DB95344B09D5 -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=Cuba -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J1 | TestReplicationHandler (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.TestReplicationHandler:
[junit4] > 1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([9F16DB95344B09D5]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=23476, name=qtp887737483-23476, state=TIMED_WAITING, group=TGRP-TestReplicationHandler]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([9F16DB95344B09D5]:0)
[junit4] Completed [379/778 (1!)] on J1 in 133.55s, 16 tests, 3 errors, 1 skipped <<< FAILURES!
[...truncated 49740 lines...]
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1690 - Still unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1690/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
2 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=5200, name=jetty-launcher-1139-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 2) Thread[id=5202, name=jetty-launcher-1139-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 3) Thread[id=5201, name=jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 4) Thread[id=5199, name=jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
1) Thread[id=5200, name=jetty-launcher-1139-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
2) Thread[id=5202, name=jetty-launcher-1139-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
3) Thread[id=5201, name=jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
4) Thread[id=5199, name=jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
at __randomizedtesting.SeedInfo.seed([2E958DD81938D690]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=5201, name=jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063) 2) Thread[id=5199, name=jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=5201, name=jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
2) Thread[id=5199, name=jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
at __randomizedtesting.SeedInfo.seed([2E958DD81938D690]:0)
Build Log:
[...truncated 12350 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_2E958DD81938D690-001/init-core-data-001
[junit4] 2> 523974 WARN (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
[junit4] 2> 523974 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 523975 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 523975 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 524037 WARN (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.h.u.NativeCodeLoader Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
[junit4] 2> 524147 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_2E958DD81938D690-001/tempDir-001
[junit4] 2> 524148 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 524148 INFO (Thread-1189) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 524148 INFO (Thread-1189) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 524150 ERROR (Thread-1189) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 524248 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:64910
[junit4] 2> 524251 INFO (zkConnectionManagerCallback-1142-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524257 INFO (jetty-launcher-1139-thread-2) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T07:27:37+10:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 524257 INFO (jetty-launcher-1139-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T07:27:37+10:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 524258 INFO (jetty-launcher-1139-thread-2) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 524258 INFO (jetty-launcher-1139-thread-2) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 524258 INFO (jetty-launcher-1139-thread-2) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 524259 INFO (jetty-launcher-1139-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@312b8388{/solr,null,AVAILABLE}
[junit4] 2> 524259 INFO (jetty-launcher-1139-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@1de3b53d{HTTP/1.1,[http/1.1]}{127.0.0.1:60307}
[junit4] 2> 524259 INFO (jetty-launcher-1139-thread-2) [ ] o.e.j.s.Server Started @526996ms
[junit4] 2> 524259 INFO (jetty-launcher-1139-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60307}
[junit4] 2> 524259 ERROR (jetty-launcher-1139-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 524259 INFO (jetty-launcher-1139-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 524259 INFO (jetty-launcher-1139-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.e.j.s.session Scavenging every 660000ms
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-02-20T21:46:08.978Z
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@3034aeaf{/solr,null,AVAILABLE}
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5fb98d50{HTTP/1.1,[http/1.1]}{127.0.0.1:46001}
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.e.j.s.Server Started @526997ms
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46001}
[junit4] 2> 524260 ERROR (jetty-launcher-1139-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 524260 INFO (jetty-launcher-1139-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-02-20T21:46:08.978Z
[junit4] 2> 524263 INFO (zkConnectionManagerCallback-1144-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524263 INFO (zkConnectionManagerCallback-1146-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524263 INFO (jetty-launcher-1139-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 524264 INFO (jetty-launcher-1139-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 524276 INFO (jetty-launcher-1139-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64910/solr
[junit4] 2> 524278 INFO (jetty-launcher-1139-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64910/solr
[junit4] 2> 524281 INFO (zkConnectionManagerCallback-1154-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524282 INFO (zkConnectionManagerCallback-1150-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524286 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524295 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524381 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 524382 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46001_solr
[junit4] 2> 524384 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer Overseer (id=72233262667268102-127.0.0.1:46001_solr-n_0000000000) starting
[junit4] 2> 524394 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46001_solr
[junit4] 2> 524410 INFO (OverseerStateUpdate-72233262667268102-127.0.0.1:46001_solr-n_0000000000) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 524420 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 524425 WARN (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 524446 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 524452 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 524456 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 524459 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60307_solr
[junit4] 2> 524461 INFO (zkCallback-1155-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 524461 INFO (zkCallback-1157-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 524486 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 524486 WARN (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 524487 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 524538 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 524538 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 524562 INFO (jetty-launcher-1139-thread-2-EventThread) [n:127.0.0.1:60307_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 524562 INFO (jetty-launcher-1139-thread-1-EventThread) [n:127.0.0.1:46001_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 524736 WARN (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 524745 WARN (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 524790 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46001.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ec4ca46
[junit4] 2> 524791 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60307.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ec4ca46
[junit4] 2> 524796 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46001.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ec4ca46
[junit4] 2> 524796 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46001.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ec4ca46
[junit4] 2> 524796 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60307.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ec4ca46
[junit4] 2> 524797 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60307.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ec4ca46
[junit4] 2> 524797 INFO (jetty-launcher-1139-thread-1) [n:127.0.0.1:46001_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_2E958DD81938D690-001/tempDir-001/node1/.
[junit4] 2> 524797 INFO (jetty-launcher-1139-thread-2) [n:127.0.0.1:60307_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_2E958DD81938D690-001/tempDir-001/node2/.
[junit4] 2> 524835 INFO (zkConnectionManagerCallback-1164-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 524862 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
[junit4] 2> 525209 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
[junit4] 2> 525239 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
[junit4] 2> 525252 WARN (qtp519964300-5063) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 525297 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 525297 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 525298 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
[junit4] 2> 525315 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
[junit4] 2> 525317 INFO (zkConnectionManagerCallback-1167-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525944 INFO (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 525951 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 525953 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64910/solr ready
[junit4] 2> 525955 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 525961 INFO (OverseerThreadFactory-2385-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection forwardingCollection
[junit4] 2> 526074 INFO (OverseerStateUpdate-72233262667268102-127.0.0.1:46001_solr-n_0000000000) [n:127.0.0.1:46001_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"forwardingCollection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"forwardingCollection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:46001/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 526285 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 526286 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 526394 INFO (zkCallback-1157-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 526395 INFO (zkCallback-1157-thread-2-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 527351 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 527376 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica_n1] Schema name=test
[junit4] 2> 527475 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 527497 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica_n1' using configuration from collection forwardingCollection, trusted=true
[junit4] 2> 527497 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_46001.solr.core.forwardingCollection.shard1.replica_n1' (registry 'solr.core.forwardingCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ec4ca46
[junit4] 2> 527498 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 527498 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_2E958DD81938D690-001/tempDir-001/node1/forwardingCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_2E958DD81938D690-001/tempDir-001/node1/./forwardingCollection_shard1_replica_n1/data/]
[junit4] 2> 527499 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=49, maxMergeAtOnceExplicit=10, maxMergedSegmentMB=99.662109375, floorSegmentMB=0.2451171875, forceMergeDeletesPctAllowed=17.459427285511417, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.27276914099242633
[junit4] 2> 527557 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 527557 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 527558 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 527558 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 527560 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=401911780, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
[junit4] 2> 527560 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7bf6244a[forwardingCollection_shard1_replica_n1] main]
[junit4] 2> 527562 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 527566 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 527583 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 527583 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1592958042558693376
[junit4] 2> 527583 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 527584 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 527584 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 527584 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 527584 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 527584 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 527585 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 527585 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 527585 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 527586 INFO (searcherExecutor-2390-thread-1-processing-n:127.0.0.1:46001_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] Registered new searcher Searcher@7bf6244a[forwardingCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 527593 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 527594 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 527594 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46001/solr/forwardingCollection_shard1_replica_n1/
[junit4] 2> 527594 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 527594 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:46001/solr/forwardingCollection_shard1_replica_n1/ has no replicas
[junit4] 2> 527594 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 527598 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46001/solr/forwardingCollection_shard1_replica_n1/ shard1
[junit4] 2> 527701 INFO (zkCallback-1157-thread-3-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 527701 INFO (zkCallback-1157-thread-2-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 527758 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 527761 INFO (qtp519964300-5067) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1476
[junit4] 2> 527765 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 527870 INFO (zkCallback-1157-thread-2-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 527870 INFO (zkCallback-1157-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 527969 INFO (OverseerCollectionConfigSetProcessor-72233262667268102-127.0.0.1:46001_solr-n_0000000000) [n:127.0.0.1:46001_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 528766 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=2812
[junit4] 2> 528767 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
[junit4] 2> 528768 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
[junit4] 2> 528773 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 528777 INFO (qtp2250314-5053) [n:127.0.0.1:60307_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 528780 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 528782 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
[junit4] 2> 528816 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
[junit4] 2> 528818 WARN (qtp519964300-5066) [ ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 528819 INFO (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 528819 INFO (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 528819 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
[junit4] 2> 528843 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
[junit4] 2> 528844 WARN (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 528862 WARN (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
[junit4] 2> id: invalid user name: "bar"
[junit4] 2>
[junit4] 2> 528867 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
[junit4] 2> 528899 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
[junit4] 2> 528900 WARN (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 550545 WARN (jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910)) [n:127.0.0.1:60307_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 21891ms for sessionid 0x1009fc50a400005
[junit4] 2> 550545 WARN (jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910)) [n:127.0.0.1:46001_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 25904ms for sessionid 0x1009fc50a400008
[junit4] 2> 550545 WARN (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]-SendThread(127.0.0.1:64910)) [ ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 21902ms for sessionid 0x1009fc50a40000b
[junit4] 2> 550545 WARN (jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910)) [n:127.0.0.1:46001_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 22700ms for sessionid 0x1009fc50a400006
[junit4] 2> 550548 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
[junit4] 2> 550572 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
[junit4] 2> 550573 WARN (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 550573 INFO (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrDispatchFilter Error authenticating
[junit4] 2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:454)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:349)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:530)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
[junit4] 2> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 550574 WARN (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.e.j.s.HttpChannel /solr/admin/collections
[junit4] 2> org.apache.solr.common.SolrException: Error during request authentication,
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:460)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:349)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1637)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:533)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:1595)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextHandle(ScopedHandler.java:188)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1253)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:168)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:473)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:1564)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.nextScope(ScopedHandler.java:166)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1155)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:527)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:132)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:530)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:347)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:256)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:279)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:102)
[junit4] 2> at org.eclipse.jetty.io.ChannelEndPoint$2.run(ChannelEndPoint.java:124)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:247)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:140)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:382)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:454)
[junit4] 2> ... 31 more
[junit4] 2> 550583 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
[junit4] 2> 550605 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
[junit4] 2> 550607 WARN (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 550608 INFO (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 550608 INFO (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 550608 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
[junit4] 2> 550632 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
[junit4] 2> 550633 WARN (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 550636 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
[junit4] 2> 550647 WARN (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@31275124 name: ZooKeeperConnection Watcher:127.0.0.1:64910/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 550647 WARN (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 550647 INFO (jetty-launcher-1139-thread-1-EventThread) [n:127.0.0.1:46001_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 550656 WARN (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7ff1b7ab name: ZooKeeperConnection Watcher:127.0.0.1:64910/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 550656 WARN (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 550656 WARN (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5abea9d8 name: ZooKeeperConnection Watcher:127.0.0.1:64910/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 550656 WARN (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 550657 INFO (jetty-launcher-1139-thread-2-EventThread) [n:127.0.0.1:60307_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 550673 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
[junit4] 2> 550673 WARN (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 565700 WARN (jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910)) [n:127.0.0.1:60307_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1009fc50a400005 has expired
[junit4] 2> 565700 WARN (jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910)) [n:127.0.0.1:60307_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1009fc50a400007 has expired
[junit4] 2> 565700 WARN (jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910)) [n:127.0.0.1:46001_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1009fc50a400008 has expired
[junit4] 2> 565700 WARN (jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910)) [n:127.0.0.1:46001_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1009fc50a400006 has expired
[junit4] 2> 565701 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
[junit4] 2> 565703 WARN (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5abea9d8 name: ZooKeeperConnection Watcher:127.0.0.1:64910/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 565703 WARN (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 565703 WARN (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7ff1b7ab name: ZooKeeperConnection Watcher:127.0.0.1:64910/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 565703 WARN (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 565703 WARN (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 565703 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer Overseer (id=72233262667268102-127.0.0.1:46001_solr-n_0000000000) closing
[junit4] 2> 565704 INFO (OverseerCollectionConfigSetProcessor-72233262667268102-127.0.0.1:46001_solr-n_0000000000) [n:127.0.0.1:46001_solr ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=72233262667268102-127.0.0.1:46001_solr-n_0000000000) am no longer a leader.
[junit4] 2> 565704 WARN (OverseerAutoScalingTriggerThread-72233262667268102-127.0.0.1:46001_solr-n_0000000000) [n:127.0.0.1:46001_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 565704 INFO (OverseerStateUpdate-72233262667268102-127.0.0.1:46001_solr-n_0000000000) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer According to ZK I (id=72233262667268102-127.0.0.1:46001_solr-n_0000000000) am no longer a leader.
[junit4] 2> 565704 INFO (OverseerStateUpdate-72233262667268102-127.0.0.1:46001_solr-n_0000000000) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46001_solr
[junit4] 2> 565706 WARN (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[2E958DD81938D690]-SendThread(127.0.0.1:64910)) [ ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1009fc50a40000b has expired
[junit4] 2> 565713 WARN (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@31275124 name: ZooKeeperConnection Watcher:127.0.0.1:64910/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 565716 WARN (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 565718 WARN (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 565718 ERROR (OverseerExitThread) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:306)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 565717 WARN (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 565721 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr-EventThread) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 565724 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 565724 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 565728 INFO (zkConnectionManagerCallback-1170-thread-1-EventThread) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 565730 INFO (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 565733 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr-EventThread) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 565736 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 565736 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 565741 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
[junit4] 2> 565741 WARN (qtp519964300-5063) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 565742 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 565742 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 565743 INFO (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 565746 INFO (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 565746 INFO (zkConnectionManagerCallback-1170-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 565748 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
[junit4] 2> 565748 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:46001_solr
[junit4] 2> 565765 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer Overseer (id=72233262667268110-127.0.0.1:46001_solr-n_0000000002) starting
[junit4] 2> 565778 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60307_solr
[junit4] 2> 565784 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 565784 INFO (zkConnectionManagerCallback-1156-thread-1-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 565784 INFO (zkCallback-1169-thread-3) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 565785 INFO (zkCallback-1155-thread-2-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 565788 INFO (zkCallback-1157-thread-14-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 565791 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
[junit4] 2> 565792 WARN (qtp519964300-5066) [n:127.0.0.1:46001_solr ] o.a.h.s.a.s.AuthenticationFilter AuthenticationToken ignored: Invalid AuthenticationToken type
[junit4] 2> 565794 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[2E958DD81938D690]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
[junit4] 2> 565794 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46001_solr
[junit4] 2> 565795 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 565795 INFO (zkConnectionManagerCallback-1158-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 565798 INFO (OverseerStateUpdate-72233262667268110-127.0.0.1:46001_solr-n_0000000002) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 565804 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.ZkController Registering core forwardingCollection_shard1_replica_n1 afterExpiration? true
[junit4] 2> 565806 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 565806 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 565806 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:46001/solr/forwardingCollection_shard1_replica_n1/
[junit4] 2> 565806 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 565806 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:46001/solr/forwardingCollection_shard1_replica_n1/ has no replicas
[junit4] 2> 565806 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 565808 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:46001/solr/forwardingCollection_shard1_replica_n1/ shard1
[junit4] 2> 565808 INFO (coreZkRegister-2375-thread-1-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 565810 INFO (zkCallback-1157-thread-11-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 565811 INFO (Thread-1214) [n:127.0.0.1:46001_solr ] o.a.s.c.SolrCore config update listener called for core forwardingCollection_shard1_replica_n1
[junit4] 2> 565812 INFO (zkCallback-1155-thread-2-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 565914 INFO (zkCallback-1157-thread-14-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 567298 WARN (zkCallback-1169-thread-3) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 567300 INFO (jetty-closer-1140-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@1de3b53d{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 567301 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=877885218
[junit4] 2> 567301 INFO (jetty-closer-1140-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@5fb98d50{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 567302 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1232648090
[junit4] 2> 567302 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 567302 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 567302 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@40187800: rootName = solr_60307, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@503b0126
[junit4] 2> 567302 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6c063c06: rootName = solr_46001, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@f094dee
[junit4] 2> 567313 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 567313 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@272479a9: rootName = solr_46001, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@457d5d4e
[junit4] 2> 567314 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 567314 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5b2037b8: rootName = solr_60307, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@457d5d4e
[junit4] 2> 567328 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 567328 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@72c52637: rootName = solr_46001, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@23bb3b36
[junit4] 2> 567329 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 567329 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4854fe7d: rootName = solr_60307, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@23bb3b36
[junit4] 2> 567330 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:46001_solr
[junit4] 2> 567331 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:60307_solr
[junit4] 2> 567331 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 567332 INFO (jetty-closer-1140-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 567332 INFO (zkCallback-1155-thread-2-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 567332 INFO (zkCallback-1157-thread-14-processing-n:127.0.0.1:46001_solr) [n:127.0.0.1:46001_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 567334 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@3ced57d4
[junit4] 2> 567334 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.forwardingCollection.shard1.replica_n1, tag=1022187476
[junit4] 2> 567334 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@410c5403: rootName = solr_46001, domain = solr.core.forwardingCollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.forwardingCollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@4158e435
[junit4] 2> 567334 INFO (zkCallback-1155-thread-3-processing-n:127.0.0.1:60307_solr) [n:127.0.0.1:60307_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 567343 ERROR (Thread[Thread-1206,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:60307_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 567350 INFO (coreCloseExecutor-2399-thread-1) [n:127.0.0.1:46001_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.forwardingCollection.shard1.leader, tag=1022187476
[junit4] 2> 567354 INFO (jetty-closer-1140-thread-1) [ ] o.a.s.c.Overseer Overseer (id=72233262667268110-127.0.0.1:46001_solr-n_0000000002) closing
[junit4] 2> 567355 INFO (OverseerStateUpdate-72233262667268110-127.0.0.1:46001_solr-n_0000000002) [n:127.0.0.1:46001_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:46001_solr
[junit4] 2> 567355 WARN (OverseerAutoScalingTriggerThread-72233262667268110-127.0.0.1:46001_solr-n_0000000002) [n:127.0.0.1:46001_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 567362 INFO (jetty-closer-1140-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@312b8388{/solr,null,UNAVAILABLE}
[junit4] 2> 567363 INFO (jetty-closer-1140-thread-2) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 567367 ERROR (Thread[Thread-1207,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:46001_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 567371 INFO (jetty-closer-1140-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@3034aeaf{/solr,null,UNAVAILABLE}
[junit4] 2> 567373 INFO (jetty-closer-1140-thread-1) [ ] o.e.j.s.session Stopped scavenging
[junit4] 2> 567374 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 567374 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64910 64910
[junit4] 2> 577910 INFO (Thread-1189) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64910 64910
[junit4] 2> 577912 WARN (Thread-1189) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/aliases.json
[junit4] 2> 6 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 3 /solr/collections/forwardingCollection/terms/shard1
[junit4] 2> 2 /solr/configs/conf1
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/clusterstate.json
[junit4] 2> 6 /solr/collections/forwardingCollection/state.json
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/collections
[junit4] 2> 4 /solr/live_nodes
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 577938 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[2E958DD81938D690]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Feb 20, 2018 9:47:02 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> 580677 ERROR (jetty-launcher-1139-thread-1-EventThread) [n:127.0.0.1:46001_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.IllegalStateException: Client is not started
[junit4] 2> at com.google.common.base.Preconditions.checkState(Preconditions.java:149)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:113)
[junit4] 2> at org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:477)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceFacade.getZooKeeper(NamespaceFacade.java:112)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:302)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:291)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:107)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2> 580697 ERROR (jetty-launcher-1139-thread-2-EventThread) [n:127.0.0.1:60307_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.IllegalStateException: Client is not started
[junit4] 2> at com.google.common.base.Preconditions.checkState(Preconditions.java:149)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.getZooKeeper(CuratorZookeeperClient.java:113)
[junit4] 2> at org.apache.curator.framework.imps.CuratorFrameworkImpl.getZooKeeper(CuratorFrameworkImpl.java:477)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceFacade.getZooKeeper(NamespaceFacade.java:112)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:302)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBui
[...truncated too long message...]
uptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] 2> 600645 WARN (jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910)) [n:127.0.0.1:60307_solr ] o.a.z.c.StaticHostProvider Unexpected exception
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] 2> 600646 WARN (jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910)) [n:127.0.0.1:46001_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> 600646 WARN (jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910)) [n:127.0.0.1:60307_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1144)
[junit4] 2> Feb 20, 2018 9:47:25 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=5201, name=jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] 2> 2) Thread[id=5199, name=jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=771, maxMBSortInHeap=6.688144081680495, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@76841865), locale=en, timezone=Europe/Samara
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=3,free=47273664,total=208896000
[junit4] 2> NOTE: All tests run in this JVM: [ZkShardTermsTest, AnalyticsQueryTest, ResponseHeaderTest, TestZkChroot, RAMDirectoryFactoryTest, QueryResultKeyTest, TestMaxScoreQueryParser, TestSolrFieldCacheBean, ClassificationUpdateProcessorTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsBasicDistributedZk2Test, LeaderElectionTest, TestInitQParser, TestUnifiedSolrHighlighter, TestCorePropertiesReload, StatelessScriptUpdateProcessorFactoryTest, TestManagedSynonymGraphFilterFactory, BinaryUpdateRequestHandlerTest, TestDynamicFieldCollectionResource, SpellCheckCollatorWithCollapseTest, PKIAuthenticationIntegrationTest, NodeLostTriggerTest, ExplicitHLLTest, TlogReplayBufferedWhileIndexingTest, InfoHandlerTest, TestSegmentSorting, DistributedVersionInfoTest, SolrMetricReporterTest, TestMergePolicyConfig, HdfsWriteToMultipleCollectionsTest, TestIntervalFaceting, LeaderElectionIntegrationTest, AddReplicaTest, TestFieldSortValues, TestCoreBackup, DistanceUnitsTest, TestIBSimilarityFactory, AnalysisAfterCoreReloadTest, TestRecovery, TestMiniSolrCloudClusterSSL, DistributedMLTComponentTest, DistributedFacetPivotLargeTest, DateMathParserTest, TestComputePlanAction, CustomHighlightComponentTest, TestPayloadScoreQParserPlugin, BJQParserTest, TestSizeLimitedDistributedMap, SpatialHeatmapFacetsTest, TestSimDistributedQueue, TestMultiWordSynonyms, SuggesterTSTTest, SolrSlf4jReporterTest, TestPushWriter, SharedFSAutoReplicaFailoverTest, UninvertDocValuesMergePolicyTest, UniqFieldsUpdateProcessorFactoryTest, QueryEqualityTest, TestBinaryField, TestConfigOverlay, DeleteShardTest, TestSolrCloudWithSecureImpersonation]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=2E958DD81938D690 -Dtests.slow=true -Dtests.locale=en -Dtests.timezone=Europe/Samara -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] > 1) Thread[id=5200, name=jetty-launcher-1139-thread-1-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
[junit4] > 2) Thread[id=5202, name=jetty-launcher-1139-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
[junit4] > 3) Thread[id=5201, name=jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > 4) Thread[id=5199, name=jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > at __randomizedtesting.SeedInfo.seed([2E958DD81938D690]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=5201, name=jetty-launcher-1139-thread-2-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > 2) Thread[id=5199, name=jetty-launcher-1139-thread-1-SendThread(127.0.0.1:64910), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > at __randomizedtesting.SeedInfo.seed([2E958DD81938D690]:0)
[junit4] Completed [117/778 (1!)] on J0 in 77.20s, 12 tests, 2 errors <<< FAILURES!
[...truncated 50602 lines...]
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1689 - Failure!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1689/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=14798, name=jetty-launcher-3038-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502) 2) Thread[id=14715, name=jetty-launcher-3038-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506) 3) Thread[id=14797, name=jetty-launcher-3038-thread-2-SendThread(127.0.0.1:64340), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
1) Thread[id=14798, name=jetty-launcher-3038-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
2) Thread[id=14715, name=jetty-launcher-3038-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
3) Thread[id=14797, name=jetty-launcher-3038-thread-2-SendThread(127.0.0.1:64340), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
at __randomizedtesting.SeedInfo.seed([16086DDE281B19EE]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=14797, name=jetty-launcher-3038-thread-2-SendThread(127.0.0.1:64340), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105) at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=14797, name=jetty-launcher-3038-thread-2-SendThread(127.0.0.1:64340), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at java.lang.Thread.sleep(Native Method)
at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
at __randomizedtesting.SeedInfo.seed([16086DDE281B19EE]:0)
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate
Error Message:
expected:<10> but was:<9>
Stack Trace:
java.lang.AssertionError: expected:<10> but was:<9>
at __randomizedtesting.SeedInfo.seed([16086DDE281B19EE:4B407357E7DDBFA1]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.failNotEquals(Assert.java:647)
at org.junit.Assert.assertEquals(Assert.java:128)
at org.junit.Assert.assertEquals(Assert.java:472)
at org.junit.Assert.assertEquals(Assert.java:456)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testSearchRate(TestLargeCluster.java:555)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12179 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
[junit4] 2> 534569 INFO (SUITE-TestLargeCluster-seed#[16086DDE281B19EE]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.sim.TestLargeCluster_16086DDE281B19EE-001/init-core-data-001
[junit4] 2> 534570 WARN (SUITE-TestLargeCluster-seed#[16086DDE281B19EE]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=21 numCloses=21
[junit4] 2> 534570 INFO (SUITE-TestLargeCluster-seed#[16086DDE281B19EE]-worker) [ ] o.a.s.SolrTestCaseJ4 Using TrieFields (NUMERIC_POINTS_SYSPROP=false) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 534572 INFO (SUITE-TestLargeCluster-seed#[16086DDE281B19EE]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 534676 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 534677 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 0
[junit4] 2> 534677 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10000_solr]
[junit4] 2> 534677 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion -1
[junit4] 2> 534677 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 0
[junit4] 2> 534679 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 534679 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 0, lastZnodeVersion 0
[junit4] 2> 534680 DEBUG (ScheduledTrigger-1347-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 48
[junit4] 2> 534689 INFO (TEST-TestLargeCluster.testBasic-seed#[16086DDE281B19EE]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testBasic
[junit4] 2> 534689 DEBUG (TEST-TestLargeCluster.testBasic-seed#[16086DDE281B19EE]) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 534690 INFO (TEST-TestLargeCluster.testBasic-seed#[16086DDE281B19EE]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 534690 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 534692 DEBUG (TEST-TestLargeCluster.testBasic-seed#[16086DDE281B19EE]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 534692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 534692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 534692 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 534693 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion -1
[junit4] 2> 534693 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 534693 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 534693 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 534693 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534713 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534734 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534755 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534775 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534795 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534800 DEBUG (simCloudManagerPool-1349-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 534800 DEBUG (simCloudManagerPool-1349-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 534801 DEBUG (simCloudManagerPool-1349-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 534801 DEBUG (simCloudManagerPool-1349-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 534801 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 534801 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 534801 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 534801 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534804 DEBUG (simCloudManagerPool-1349-thread-2) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 534804 DEBUG (simCloudManagerPool-1349-thread-2) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 534805 DEBUG (simCloudManagerPool-1349-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 534805 DEBUG (simCloudManagerPool-1349-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 534805 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 534805 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 534805 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 534821 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534823 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534841 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534844 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534867 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534867 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534887 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534887 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534908 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534908 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534928 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534929 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534949 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534949 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534969 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534969 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 534989 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 534989 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535010 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535010 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535030 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535030 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535050 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535050 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535070 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535070 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535090 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535090 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535111 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535111 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535131 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535131 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535151 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535151 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535173 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535174 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535194 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535194 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535214 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535214 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535234 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535235 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535255 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535255 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535275 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535275 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535296 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535296 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535319 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535319 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535339 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535340 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535360 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535360 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535380 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535380 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535400 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535400 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535421 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535421 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535444 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535444 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535464 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535465 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535485 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535485 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535505 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535505 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535525 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535525 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535546 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535546 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535566 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535566 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535587 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535587 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535607 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535607 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535627 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535627 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535647 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535647 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535667 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535667 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535688 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535688 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535708 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535708 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535728 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535729 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535749 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 100
[junit4] 2> 535750 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 535764 DEBUG (simCloudManagerPool-1349-thread-79) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard2: {"core_node18":{
[junit4] 2> "node_name":"127.0.0.1:10038_solr",
[junit4] 2> "core":"testBasic_shard2_replica_n18",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":1000000000,
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 535764 DEBUG (simCloudManagerPool-1349-thread-83) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard1: {"core_node1":{
[junit4] 2> "node_name":"127.0.0.1:10057_solr",
[junit4] 2> "core":"testBasic_shard1_replica_n1",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":1000000000,
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 535765 DEBUG (simCloudManagerPool-1349-thread-81) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard4: {"core_node54":{
[junit4] 2> "node_name":"127.0.0.1:10018_solr",
[junit4] 2> "core":"testBasic_shard4_replica_t54",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":1000000000,
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 535766 DEBUG (simCloudManagerPool-1349-thread-82) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard5: {"core_node65":{
[junit4] 2> "node_name":"127.0.0.1:10026_solr",
[junit4] 2> "core":"testBasic_shard5_replica_n65",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":1000000000,
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 535766 DEBUG (simCloudManagerPool-1349-thread-80) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testBasic / shard3: {"core_node39":{
[junit4] 2> "node_name":"127.0.0.1:10060_solr",
[junit4] 2> "core":"testBasic_shard3_replica_t39",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":1000000000,
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic"}}
[junit4] 2> 535766 INFO (TEST-TestLargeCluster.testBasic-seed#[16086DDE281B19EE]) [ ] o.a.s.c.a.s.TestLargeCluster Ready after 823ms
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 96
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10021_solr
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10006_solr
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10064_solr
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10027_solr
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 96
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10021_solr
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10006_solr
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10064_solr
[junit4] 2> 535772 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10027_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10005_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10033_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10033_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10023_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10023_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10048_solr
[junit4] 2> 535792 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10048_solr
[junit4] 2> 535813 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 535813 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 535833 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 535833 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 535854 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 535854 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 535874 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 535874 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 535874 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10006_solr, 127.0.0.1:10021_solr, 127.0.0.1:10027_solr, 127.0.0.1:10064_solr]
[junit4] 2> 535874 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"968161d4a7592T6iffm5zfshajoxyl2agrxpi4q",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647718980384146,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647718980384146,
[junit4] 2> 2647718979393946,
[junit4] 2> 2647718982073246,
[junit4] 2> 2647718981233946],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 535875 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 535878 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"968161d4a7592T6iffm5zfshajoxyl2agrxpi4q",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647718980384146,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647718980384146,
[junit4] 2> 2647718979393946,
[junit4] 2> 2647718982073246,
[junit4] 2> 2647718981233946],
[junit4] 2> "_enqueue_time_":2647724153691146,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 535878 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"968161d4a7592T6iffm5zfshajoxyl2agrxpi4q",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647718980384146,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647718980384146,
[junit4] 2> 2647718979393946,
[junit4] 2> 2647718982073246,
[junit4] 2> 2647718981233946],
[junit4] 2> "_enqueue_time_":2647724153691146,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
[junit4] 2> 535901 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node28
[junit4] 2> 535911 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node68
[junit4] 2> 535918 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node24
[junit4] 2> 535921 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 535921 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2647726438 sessionWrapper.createTime 2647724410664996, this.sessionWrapper.createTime 2647724410664996
[junit4] 2> 535921 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"968161d4a7592T6iffm5zfshajoxyl2agrxpi4q",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647718980384146,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647718980384146,
[junit4] 2> 2647718979393946,
[junit4] 2> 2647718982073246,
[junit4] 2> 2647718981233946],
[junit4] 2> "_enqueue_time_":2647724153691146,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@55786e17, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@5b3cc74d, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@64e7bc02, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@23494bf0], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
[junit4] 2> 535921 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10072_solr&inPlaceMove=true&replica=core_node28
[junit4] 2> 536028 DEBUG (simCloudManagerPool-1349-thread-125) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node77":{
[junit4] 2> "core":"testBasic_shard2_replica_p76",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10072_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 536142 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10056_solr&inPlaceMove=true&replica=core_node68
[junit4] 2> 536247 DEBUG (simCloudManagerPool-1349-thread-133) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node79":{
[junit4] 2> "core":"testBasic_shard5_replica_t78",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10056_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 536366 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10059_solr&inPlaceMove=true&replica=core_node24
[junit4] 2> 536467 DEBUG (simCloudManagerPool-1349-thread-141) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node81":{
[junit4] 2> "core":"testBasic_shard2_replica_t80",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10059_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 536584 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10053_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 536687 DEBUG (simCloudManagerPool-1349-thread-149) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node83":{
[junit4] 2> "core":"testBasic_shard1_replica_n82",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10053_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 536802 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 536802 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
[junit4] 2> 536802 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 46220 ms for event id=968161d4a7592T6iffm5zfshajoxyl2agrxpi4q
[junit4] 2> 536902 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 92
[junit4] 2> 536902 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 92
[junit4] 2> 536902 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10033_solr, 127.0.0.1:10005_solr, 127.0.0.1:10023_solr, 127.0.0.1:10048_solr]
[junit4] 2> 536902 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10006_solr, 127.0.0.1:10033_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10027_solr, 127.0.0.1:10023_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr]
[junit4] 2> 536902 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"968165a45d068T6iffm5zfshajoxyl2agrxpi4t",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647720003489896,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720003489896,
[junit4] 2> 2647720001851446,
[junit4] 2> 2647720004910096,
[junit4] 2> 2647720011113996],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10048_solr"]}}
[junit4] 2> 536902 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 536902 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"968161e01d2c8T6iffm5zfshajoxyl2agrxpi4u",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":2647718992401096,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647718992401096,
[junit4] 2> 2647720004315796,
[junit4] 2> 2647718991536296,
[junit4] 2> 2647720002897596,
[junit4] 2> 2647718994175946,
[junit4] 2> 2647720005705296,
[junit4] 2> 2647720013902596,
[junit4] 2> 2647718993242046],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10048_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 536903 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"968165a45d068T6iffm5zfshajoxyl2agrxpi4t",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647720003489896,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720003489896,
[junit4] 2> 2647720001851446,
[junit4] 2> 2647720004910096,
[junit4] 2> 2647720011113996],
[junit4] 2> "_enqueue_time_":2647775530646446,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10048_solr"]}}
[junit4] 2> 536904 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:10006_solr, 127.0.0.1:10033_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10027_solr, 127.0.0.1:10023_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr] is not ready, will try later
[junit4] 2> 536904 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"968165a45d068T6iffm5zfshajoxyl2agrxpi4t",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647720003489896,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720003489896,
[junit4] 2> 2647720001851446,
[junit4] 2> 2647720004910096,
[junit4] 2> 2647720011113996],
[junit4] 2> "_enqueue_time_":2647775530646446,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10048_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
[junit4] 2> 536913 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node53
[junit4] 2> 536919 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10094_solr&inPlaceMove=true&replica=core_node62
[junit4] 2> 536923 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10091_solr&inPlaceMove=true&replica=core_node30
[junit4] 2> 536925 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10034_solr&inPlaceMove=true&replica=core_node41
[junit4] 2> 536925 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2647776676 sessionWrapper.createTime 2647775689546596, this.sessionWrapper.createTime 2647775689546596
[junit4] 2> 536925 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"968165a45d068T6iffm5zfshajoxyl2agrxpi4t",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647720003489896,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720003489896,
[junit4] 2> 2647720001851446,
[junit4] 2> 2647720004910096,
[junit4] 2> 2647720011113996],
[junit4] 2> "_enqueue_time_":2647775530646446,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10048_solr"]}} with context properties: {operations=[org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@79ec3618, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@541f4a6c, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@6f7ea0e7, org.apache.solr.client.solrj.request.CollectionAdminRequest$MoveReplica@12a408a6], AFTER_ACTION=[compute], BEFORE_ACTION=[compute, execute]}
[junit4] 2> 536926 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10037_solr&inPlaceMove=true&replica=core_node53
[junit4] 2> 537027 DEBUG (simCloudManagerPool-1349-thread-160) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node85":{
[junit4] 2> "core":"testBasic_shard4_replica_t84",
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10037_solr",
[junit4] 2> "type":"TLOG"}}
[junit4] 2> 537138 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10094_solr&inPlaceMove=true&replica=core_node62
[junit4] 2> 537244 DEBUG (simCloudManagerPool-1349-thread-168) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node87":{
[junit4] 2> "core":"testBasic_shard5_replica_n86",
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10094_solr",
[junit4] 2> "type":"NRT"}}
[junit4] 2> 537366 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10091_solr&inPlaceMove=true&replica=core_node30
[junit4] 2> 537468 DEBUG (simCloudManagerPool-1349-thread-176) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node89":{
[junit4] 2> "core":"testBasic_shard2_replica_p88",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10091_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 537581 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction Executing operation: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10034_solr&inPlaceMove=true&replica=core_node41
[junit4] 2> 537683 DEBUG (simCloudManagerPool-1349-thread-184) [ ] o.a.s.c.a.s.SimClusterStateProvider -- new replica: {"core_node91":{
[junit4] 2> "core":"testBasic_shard3_replica_p90",
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testBasic",
[junit4] 2> "node_name":"127.0.0.1:10034_solr",
[junit4] 2> "type":"PULL"}}
[junit4] 2> 537801 INFO (TEST-TestLargeCluster.testBasic-seed#[16086DDE281B19EE]) [ ] o.a.s.c.a.s.TestLargeCluster Ready after 101146ms
[junit4] 2> 537802 INFO (TEST-TestLargeCluster.testBasic-seed#[16086DDE281B19EE]) [ ] o.a.s.c.a.s.TestLargeCluster OP COUNTS: {CREATE=1, DELETESTATUS=7, MOVEREPLICA=8, REQUESTSTATUS=8, autoscaling=2, update=4}
[junit4] 2> 537803 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 537803 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
[junit4] 2> 537803 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 44989 ms for event id=968165a45d068T6iffm5zfshajoxyl2agrxpi4t
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 82
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 82
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10074_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10074_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10090_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10090_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10087_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10087_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10009_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10098_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10009_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10098_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10030_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10011_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10030_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10054_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10011_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10055_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10054_solr
[junit4] 2> 537903 DEBUG (ScheduledTrigger-1350-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10016_solr
[junit4] 2> 537904 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10055_solr
[junit4] 2> 537904 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:10016_solr
[junit4] 2> 537904 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10033_solr, 127.0.0.1:10023_solr, 127.0.0.1:10006_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10027_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr]
[junit4] 2> 537904 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"968165a526a94T6iffm5zfshajoxyl2agrxpi4y",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":2647720004315796,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720004315796,
[junit4] 2> 2647720005705296,
[junit4] 2> 2647718992401096,
[junit4] 2> 2647718991536296,
[junit4] 2> 2647720002897596,
[junit4] 2> 2647718994175946,
[junit4] 2> 2647720013902596,
[junit4] 2> 2647718993242046],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10048_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 537904 DEBUG (ScheduledTrigger-1350-thread-4) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 537904 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"968165a526a94T6iffm5zfshajoxyl2agrxpi4y",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":2647720004315796,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720004315796,
[junit4] 2> 2647720005705296,
[junit4] 2> 2647718992401096,
[junit4] 2> 2647718991536296,
[junit4] 2> 2647720002897596,
[junit4] 2> 2647718994175946,
[junit4] 2> 2647720013902596,
[junit4] 2> 2647718993242046],
[junit4] 2> "_enqueue_time_":2647825622396246,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10048_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 537904 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"968165a526a94T6iffm5zfshajoxyl2agrxpi4y",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":2647720004315796,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720004315796,
[junit4] 2> 2647720005705296,
[junit4] 2> 2647718992401096,
[junit4] 2> 2647718991536296,
[junit4] 2> 2647720002897596,
[junit4] 2> 2647718994175946,
[junit4] 2> 2647720013902596,
[junit4] 2> 2647718993242046],
[junit4] 2> "_enqueue_time_":2647825622396246,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10048_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}} with context properties: {BEFORE_ACTION=[auto_add_replicas_plan]}
[junit4] 2> 537907 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2647825757 sessionWrapper.createTime 2647825735313346, this.sessionWrapper.createTime 2647825735313346
[junit4] 2> 537907 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction -- processing event: {
[junit4] 2> "id":"968165a526a94T6iffm5zfshajoxyl2agrxpi4y",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":2647720004315796,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720004315796,
[junit4] 2> 2647720005705296,
[junit4] 2> 2647718992401096,
[junit4] 2> 2647718991536296,
[junit4] 2> 2647720002897596,
[junit4] 2> 2647718994175946,
[junit4] 2> 2647720013902596,
[junit4] 2> 2647718993242046],
[junit4] 2> "_enqueue_time_":2647825622396246,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10048_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}} with context properties: {AFTER_ACTION=[auto_add_replicas_plan], BEFORE_ACTION=[auto_add_replicas_plan, execute_plan]}
[junit4] 2> 537907 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ExecutePlanAction No operations to execute for event: {
[junit4] 2> "id":"968165a526a94T6iffm5zfshajoxyl2agrxpi4y",
[junit4] 2> "source":".auto_add_replicas",
[junit4] 2> "eventTime":2647720004315796,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647720004315796,
[junit4] 2> 2647720005705296,
[junit4] 2> 2647718992401096,
[junit4] 2> 2647718991536296,
[junit4] 2> 2647720002897596,
[junit4] 2> 2647718994175946,
[junit4] 2> 2647720013902596,
[junit4] 2> 2647718993242046],
[junit4] 2> "_enqueue_time_":2647825622396246,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10033_solr",
[junit4] 2> "127.0.0.1:10023_solr",
[junit4] 2> "127.0.0.1:10006_solr",
[junit4] 2> "127.0.0.1:10021_solr",
[junit4] 2> "127.0.0.1:10005_solr",
[junit4] 2> "127.0.0.1:10027_solr",
[junit4] 2> "127.0.0.1:10048_solr",
[junit4] 2> "127.0.0.1:10064_solr"]}}
[junit4] 2> 537907 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: .auto_add_replicas after 100ms
[junit4] 2> 537907 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_lost_trigger after 100ms
[junit4] 2> 537907 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing took 154 ms for event id=968165a526a94T6iffm5zfshajoxyl2agrxpi4y
[junit4] 2> 538007 DEBUG (ScheduledTrigger-1350-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 82
[junit4] 2> 538007 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 82
[junit4] 2> 538008 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:10055_solr, 127.0.0.1:10016_solr, 127.0.0.1:10054_solr, 127.0.0.1:10087_solr, 127.0.0.1:10074_solr, 127.0.0.1:10009_solr, 127.0.0.1:10098_solr, 127.0.0.1:10090_solr, 127.0.0.1:10030_solr, 127.0.0.1:10011_solr]
[junit4] 2> 538008 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"9682ef09855c8T6iffm5zfshajoxyl2agrxpi51",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647825604695496,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647825604695496,
[junit4] 2> 2647825606125946,
[junit4] 2> 2647825603174846,
[junit4] 2> 2647825594030846,
[junit4] 2> 2647825587477996,
[junit4] 2> 2647825594988746,
[junit4] 2> 2647825595641646,
[junit4] 2> 2647825589199696,
[junit4] 2> 2647825600746246,
[junit4] 2> 2647825601492446],
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10055_solr",
[junit4] 2> "127.0.0.1:10016_solr",
[junit4] 2> "127.0.0.1:10054_solr",
[junit4] 2> "127.0.0.1:10087_solr",
[junit4] 2> "127.0.0.1:10074_solr",
[junit4] 2> "127.0.0.1:10009_solr",
[junit4] 2> "127.0.0.1:10098_solr",
[junit4] 2> "127.0.0.1:10090_solr",
[junit4] 2> "127.0.0.1:10030_solr",
[junit4] 2> "127.0.0.1:10011_solr"]}}
[junit4] 2> 538008 DEBUG (ScheduledTrigger-1350-thread-1) [ ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [.auto_add_replicas, node_lost_trigger]
[junit4] 2> 538009 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"9682ef09855c8T6iffm5zfshajoxyl2agrxpi51",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647825604695496,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647825604695496,
[junit4] 2> 2647825606125946,
[junit4] 2> 2647825603174846,
[junit4] 2> 2647825594030846,
[junit4] 2> 2647825587477996,
[junit4] 2> 2647825594988746,
[junit4] 2> 2647825595641646,
[junit4] 2> 2647825589199696,
[junit4] 2> 2647825600746246,
[junit4] 2> 2647825601492446],
[junit4] 2> "_enqueue_time_":2647830823552146,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10055_solr",
[junit4] 2> "127.0.0.1:10016_solr",
[junit4] 2> "127.0.0.1:10054_solr",
[junit4] 2> "127.0.0.1:10087_solr",
[junit4] 2> "127.0.0.1:10074_solr",
[junit4] 2> "127.0.0.1:10009_solr",
[junit4] 2> "127.0.0.1:10098_solr",
[junit4] 2> "127.0.0.1:10090_solr",
[junit4] 2> "127.0.0.1:10030_solr",
[junit4] 2> "127.0.0.1:10011_solr"]}}
[junit4] 2> 538009 DEBUG (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"9682ef09855c8T6iffm5zfshajoxyl2agrxpi51",
[junit4] 2> "source":"node_lost_trigger",
[junit4] 2> "eventTime":2647825604695496,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[
[junit4] 2> 2647825604695496,
[junit4] 2> 2647825606125946,
[junit4] 2> 2647825603174846,
[junit4] 2> 2647825594030846,
[junit4] 2> 2647825587477996,
[junit4] 2> 2647825594988746,
[junit4] 2> 2647825595641646,
[junit4] 2> 2647825589199696,
[junit4] 2> 2647825600746246,
[junit4] 2> 2647825601492446],
[junit4] 2> "_enqueue_time_":2647830823552146,
[junit4] 2> "nodeNames":[
[junit4] 2> "127.0.0.1:10055_solr",
[junit4] 2> "127.0.0.1:10016_solr",
[junit4] 2> "127.0.0.1:10054_solr",
[junit4] 2> "127.0.0.1:10087_solr",
[junit4] 2> "127.0.0.1:10074_solr",
[junit4] 2> "127.0.0.1:10009_solr",
[junit4] 2> "127.0.0.1:10098_solr",
[junit4] 2> "127.0.0.1:10090_solr",
[junit4] 2> "127.0.0.1:10030_solr",
[junit4] 2> "127.0.0.1:10011_solr"]}} with context properties: {BEFORE_ACTION=[compute]}
[junit4] 2> 538018 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10031_solr&inPlaceMove=true&replica=core_node16
[junit4] 2> 538029 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10097_solr&inPlaceMove=true&replica=core_node26
[junit4] 2> 538046 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10002_solr&inPlaceMove=true&replica=core_node63
[junit4] 2> 538055 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10078_solr&inPlaceMove=true&replica=core_node57
[junit4] 2> 538063 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10075_solr&inPlaceMove=true&replica=core_node33
[junit4] 2> 538067 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10050_solr&inPlaceMove=true&replica=core_node71
[junit4] 2> 538070 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10024_solr&inPlaceMove=true&replica=core_node37
[junit4] 2> 538073 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10008_solr&inPlaceMove=true&replica=core_node66
[junit4] 2> 538075 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testBasic&targetNode=127.0.0.1:10062_solr&inPlaceMove=true&replica=core_node52
[junit4] 2> 538077 INFO (AutoscalingActionExecutor-1351-thread-1) [ ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLI
[...truncated too long message...]
ntCnxn.java:1063)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1020, maxMBSortInHeap=6.275834500685828, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@456aa7c2), locale=ar-IQ, timezone=America/Anchorage
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=4,free=68938312,total=368631808
[junit4] 2> NOTE: All tests run in this JVM: [BasicAuthIntegrationTest, SSLMigrationTest, CreateCollectionCleanupTest, CursorPagingTest, TestSimpleTextCodec, DateMathParserTest, MBeansHandlerTest, ParsingFieldUpdateProcessorsTest, ResponseHeaderTest, TestConfigSetProperties, SolrRequestParserTest, DeleteInactiveReplicaTest, ComputePlanActionTest, VersionInfoTest, TestRTimerTree, TestCloudRecovery, DistribJoinFromCollectionTest, HdfsWriteToMultipleCollectionsTest, TestLeaderInitiatedRecoveryThread, MoveReplicaTest, TestSQLHandlerNonCloud, SchemaVersionSpecificBehaviorTest, HdfsAutoAddReplicasIntegrationTest, DistributedExpandComponentTest, TestTolerantSearch, TolerantUpdateProcessorTest, TestRawTransformer, BigEndianAscendingWordSerializerTest, AtomicUpdateProcessorFactoryTest, ConfigSetsAPITest, TestSolrQueryResponse, BlobRepositoryCloudTest, CoreSorterTest, TestFieldTypeCollectionResource, TestDistribDocBasedVersion, TestLargeCluster, CdcrVersionReplicationTest, WordBreakSolrSpellCheckerTest, TestPayloadScoreQParserPlugin, TermsComponentTest, SpellCheckCollatorWithCollapseTest, ReplaceNodeTest, ResourceLoaderTest, PropertiesRequestHandlerTest, TestFieldCacheVsDocValues, TestSystemIdResolver, LeaderInitiatedRecoveryOnShardRestartTest, TestQuerySenderNoQuery, ZkShardTermsTest, TestRecovery, TestExpandComponent, SynonymTokenizerTest, TestQueryUtils, MoreLikeThisHandlerTest, TestCorePropertiesReload, TestExactStatsCache, TestIBSimilarityFactory, TestPseudoReturnFields, TestSchemaNameResource, ClassificationUpdateProcessorTest, TestSolrConfigHandlerConcurrent, TestLocalFSCloudBackupRestore, ExitableDirectoryReaderTest, TestSha256AuthenticationProvider, FacetPivotSmallTest, TestRestoreCore, MigrateRouteKeyTest, TestRandomRequestDistribution, UnloadDistributedZkTest, PathHierarchyTokenizerFactoryTest, TestCollationField, AddBlockUpdateTest, SubstringBytesRefFilterTest, SuggesterTSTTest, CloudExitableDirectoryReaderTest, AutoAddReplicasPlanActionTest, TestLegacyFieldReuse, TestPullReplicaErrorHandling, BufferStoreTest, TestRequestForwarding, TestSuggestSpellingConverter, TestReRankQParserPlugin, CdcrUpdateLogTest, DocExpirationUpdateProcessorFactoryTest, TestIntervalFaceting, TestNonDefinedSimilarityFactory, TestFoldingMultitermQuery, CachingDirectoryFactoryTest, SuggesterFSTTest, SignatureUpdateProcessorFactoryTest, TestRemoteStreaming, TestScoreJoinQPScore, TestCollationFieldDocValues, OverseerModifyCollectionTest, TestConfigSetsAPIExclusivity, TestSurroundQueryParser, V2ApiIntegrationTest, TestSolrCoreSnapshots, TestNodeLostTrigger, BooleanFieldTest, TestBlobHandler, TestGroupingSearch, TestMacroExpander, CdcrBootstrapTest, TestFieldCacheSort, HdfsCollectionsAPIDistributedZkTest, RestartWhileUpdatingTest, TestRealTimeGet, TestFiltering, SolrCoreMetricManagerTest, ConcurrentDeleteAndCreateCollectionTest, BlockCacheTest, TestZkChroot, TestEmbeddedSolrServerAdminHandler, SolrShardReporterTest, TestMaxTokenLenTokenizer, StatsComponentTest, SharedFSAutoReplicaFailoverTest, TestCollectionsAPIViaSolrCloudCluster, TestLazyCores, DistributedFacetPivotWhiteBoxTest, TestClusterProperties, TestInitParams, PluginInfoTest, LeaderElectionIntegrationTest, QueryEqualityTest, TestBinaryField, TestStressInPlaceUpdates, OverseerStatusTest, TestCloudInspectUtil, CoreAdminRequestStatusTest, TestBlendedInfixSuggestions, TestSweetSpotSimilarityFactory, GraphQueryTest, RAMDirectoryFactoryTest, TestSolrDeletionPolicy1, DirectSolrConnectionTest, TestPartialUpdateDeduplication, TestLegacyTerms, UninvertDocValuesMergePolicyTest, TestLegacyFieldCache, SpatialRPTFieldTypeTest, TestTlogReplica, TestApiFramework, TestSimpleQParserPlugin, TestPrepRecovery, MoveReplicaHDFSFailoverTest, HdfsNNFailoverTest, TestSolrCloudWithSecureImpersonation]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=16086DDE281B19EE -Dtests.slow=true -Dtests.locale=ar-IQ -Dtests.timezone=America/Anchorage -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] > 1) Thread[id=14798, name=jetty-launcher-3038-thread-2-EventThread, state=WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:502)
[junit4] > 2) Thread[id=14715, name=jetty-launcher-3038-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] > 3) Thread[id=14797, name=jetty-launcher-3038-thread-2-SendThread(127.0.0.1:64340), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > at __randomizedtesting.SeedInfo.seed([16086DDE281B19EE]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=14797, name=jetty-launcher-3038-thread-2-SendThread(127.0.0.1:64340), state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at java.lang.Thread.sleep(Native Method)
[junit4] > at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:105)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:1000)
[junit4] > at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1063)
[junit4] > at __randomizedtesting.SeedInfo.seed([16086DDE281B19EE]:0)
[junit4] Completed [259/778 (2!)] on J0 in 63.86s, 12 tests, 2 errors <<< FAILURES!
[...truncated 10918 lines...]
-validate-source-patterns:
[source-patterns] invalid logging pattern [not private static final, uses static class name]: solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/SolrCollector.java
[source-patterns] invalid logging pattern [not private static final, uses static class name]: solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java
[source-patterns] invalid logging pattern [not private static final, uses static class name]: solr/contrib/prometheus-exporter/src/java/org/apache/solr/prometheus/scraper/SolrScraper.java
BUILD FAILED
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:618: The following error occurred while executing this line:
/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/build.xml:128: Found 3 violations in source files (invalid logging pattern [not private static final, uses static class name]).
Total time: 81 minutes 16 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
[WARNINGS] Skipping publisher since build result is FAILURE
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-master-Solaris (64bit/jdk1.8.0) - Build #
1688 - Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-master-Solaris/1688/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.handler.clustering.DistributedClusteringComponentTest
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.handler.clustering.DistributedClusteringComponentTest: 1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.clustering.DistributedClusteringComponentTest:
1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([8AEF5CD4F3FBC58]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.handler.clustering.DistributedClusteringComponentTest
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308) at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([8AEF5CD4F3FBC58]:0)
FAILED: org.apache.solr.cloud.autoscaling.ComputePlanActionTest.testNodeWithMultipleReplicasLost
Error Message:
The operations computed by ComputePlanAction should not be null SolrClientNodeStateProvider.DEBUG{AFTER_ACTION=[compute_plan, null], BEFORE_ACTION=[compute_plan, null]}
Stack Trace:
java.lang.AssertionError: The operations computed by ComputePlanAction should not be null SolrClientNodeStateProvider.DEBUG{AFTER_ACTION=[compute_plan, null], BEFORE_ACTION=[compute_plan, null]}
at __randomizedtesting.SeedInfo.seed([4324EF2CAA18D31F:73E40EAE226A3243]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNotNull(Assert.java:526)
at org.apache.solr.cloud.autoscaling.ComputePlanActionTest.testNodeWithMultipleReplicasLost(ComputePlanActionTest.java:291)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13936 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.ComputePlanActionTest
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/init-core-data-001
[junit4] 2> 3858239 WARN (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=24 numCloses=24
[junit4] 2> 3858239 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 3858241 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 3858241 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3858242 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/tempDir-001
[junit4] 2> 3858242 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3858243 INFO (Thread-9248) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3858243 INFO (Thread-9248) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3858244 ERROR (Thread-9248) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3858342 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:61117
[junit4] 2> 3858344 INFO (zkConnectionManagerCallback-6373-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3858348 INFO (jetty-launcher-6370-thread-1) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T10:27:37+13:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 3858349 INFO (jetty-launcher-6370-thread-1) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3858349 INFO (jetty-launcher-6370-thread-1) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3858349 INFO (jetty-launcher-6370-thread-1) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 3858349 INFO (jetty-launcher-6370-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@293e4c9{/solr,null,AVAILABLE}
[junit4] 2> 3858350 INFO (jetty-launcher-6370-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@15eebe3c{HTTP/1.1,[http/1.1]}{127.0.0.1:38925}
[junit4] 2> 3858350 INFO (jetty-launcher-6370-thread-1) [ ] o.e.j.s.Server Started @3865637ms
[junit4] 2> 3858350 INFO (jetty-launcher-6370-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38925}
[junit4] 2> 3858350 ERROR (jetty-launcher-6370-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3858350 INFO (jetty-launcher-6370-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 3858350 INFO (jetty-launcher-6370-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3858350 INFO (jetty-launcher-6370-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3858350 INFO (jetty-launcher-6370-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-02-20T02:25:16.592Z
[junit4] 2> 3858352 INFO (zkConnectionManagerCallback-6375-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3858353 INFO (jetty-launcher-6370-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3858359 INFO (jetty-launcher-6370-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61117/solr
[junit4] 2> 3858360 INFO (zkConnectionManagerCallback-6379-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3858363 INFO (zkConnectionManagerCallback-6381-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3858502 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3858503 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38925_solr
[junit4] 2> 3858504 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer Overseer (id=72228692014661635-127.0.0.1:38925_solr-n_0000000000) starting
[junit4] 2> 3858513 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 3858513 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38925_solr
[junit4] 2> 3858514 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 3858515 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 3858516 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: []
[junit4] 2> 3858516 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 3858516 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 3858516 INFO (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3858519 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3858520 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 3858521 DEBUG (ScheduledTrigger-12741-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 1
[junit4] 2> 3858603 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38925.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3858609 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38925.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3858609 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38925.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3858609 INFO (jetty-launcher-6370-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/tempDir-001/node1/.
[junit4] 2> 3858622 INFO (zkConnectionManagerCallback-6385-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3858626 INFO (zkConnectionManagerCallback-6389-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3858629 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3858630 INFO (SUITE-ComputePlanActionTest-seed#[4324EF2CAA18D31F]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:61117/solr ready
[junit4] 2> 3858644 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAdded
[junit4] 2> 3858645 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 3858645 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 3858646 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3858647 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3858648 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3858648 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 3858651 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3858652 DEBUG (qtp2092840606-33569) [n:127.0.0.1:38925_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3858653 INFO (qtp2092840606-33569) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 3858653 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 3858653 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 3858653 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3858654 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3858654 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 3858655 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3858655 DEBUG (qtp2092840606-33571) [n:127.0.0.1:38925_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3858656 INFO (qtp2092840606-33571) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 3858656 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 3858657 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 3858657 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3858657 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3858658 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 3858660 INFO (qtp2092840606-33571) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3858660 DEBUG (qtp2092840606-33574) [n:127.0.0.1:38925_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3858661 INFO (qtp2092840606-33574) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=2
[junit4] 2> 3858661 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
[junit4] 2> 3858662 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:38925_solr]
[junit4] 2> 3858662 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.ComputePlanActionTest$AssertingTriggerAction}], enabled=true}
[junit4] 2> 3858662 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 3858663 INFO (qtp2092840606-33574) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3858663 DEBUG (qtp2092840606-33572) [n:127.0.0.1:38925_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3858664 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 3858664 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
[junit4] 2> 3858665 INFO (qtp2092840606-33571) [n:127.0.0.1:38925_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=testNodeAdded&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3858665 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:38925_solr]
[junit4] 2> 3858665 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.ComputePlanActionTest$AssertingTriggerAction}], enabled=true}
[junit4] 2> 3858666 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3858667 INFO (OverseerThreadFactory-12743-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.c.CreateCollectionCmd Create collection testNodeAdded
[junit4] 2> 3858667 DEBUG (ScheduledTrigger-12741-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3858667 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 5
[junit4] 2> 3858667 DEBUG (ScheduledTrigger-12741-thread-1) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 3858668 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
[junit4] 2> 3858668 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3858668 WARN (OverseerThreadFactory-12743-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.c.CreateCollectionCmd Specified number of replicas of 2 on collection testNodeAdded is higher than the number of Solr instances currently live or live and part of your createNodeSet(1). It's unusual to run two replica of the same slice on the same Solr-instance.
[junit4] 2> 3858668 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 6, lastZnodeVersion 6
[junit4] 2> 3858670 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3858670 INFO (OverseerThreadFactory-12743-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2610748363 sessionWrapper.createTime 2610748362892400, this.sessionWrapper.createTime 2610748362892400
[junit4] 2> 3858676 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "name":"testNodeAdded",
[junit4] 2> "fromApi":"true",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "collection.configName":"conf",
[junit4] 2> "numShards":"1",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "stateFormat":"2",
[junit4] 2> "operation":"create"} current state version: 0
[junit4] 2> 3858677 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ClusterStateMutator building a new cName: testNodeAdded
[junit4] 2> 3858677 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/testNodeAdded/state.json
[junit4] 2> 3858782 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"} current state version: 0
[junit4] 2> 3858782 INFO (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3858785 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{}}
[junit4] 2> 3858785 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{"core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"}}}
[junit4] 2> 3858786 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"} current state version: 0
[junit4] 2> 3858786 INFO (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3858788 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.SliceMutator Old Slice: shard1:{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{"core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"}}}
[junit4] 2> 3858788 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.SliceMutator New Slice: shard1:{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"}}}
[junit4] 2> 3858889 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testNodeAdded/state.json version: 0
[junit4] 2> 3858989 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=testNodeAdded_shard1_replica_n1&action=CREATE&numShards=1&collection=testNodeAdded&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3858989 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3858989 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=testNodeAdded_shard1_replica_n2&action=CREATE&numShards=1&collection=testNodeAdded&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "core_node_name":"core_node4",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"} current state version: 0
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "core_node_name":"core_node4",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"}
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node4:{"core":"testNodeAdded_shard1_replica_n2","base_url":"http://127.0.0.1:38925/solr","node_name":"127.0.0.1:38925_solr","state":"down","type":"NRT"}
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(testNodeAdded//collections/testNodeAdded/state.json/1)={
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "state":"down",
[junit4] 2> "node_name":null,
[junit4] 2> "type":"NRT"},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"}}}}}
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "core_node_name":"core_node3",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"} current state version: 0
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "core_node_name":"core_node3",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"down",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"}
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node3:{"core":"testNodeAdded_shard1_replica_n1","base_url":"http://127.0.0.1:38925/solr","node_name":"127.0.0.1:38925_solr","state":"down","type":"NRT"}
[junit4] 2> 3858994 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(testNodeAdded//collections/testNodeAdded/state.json/1)={
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"}}}}}
[junit4] 2> 3859095 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testNodeAdded/state.json version: 1
[junit4] 2> 3859096 INFO (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3859096 INFO (zkCallback-6380-thread-2-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3859674 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3859674 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 3860016 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 3860016 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 8.0.0
[junit4] 2> 3860034 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.s.IndexSchema [testNodeAdded_shard1_replica_n1] Schema name=minimal
[junit4] 2> 3860034 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.s.IndexSchema [testNodeAdded_shard1_replica_n2] Schema name=minimal
[junit4] 2> 3860036 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3860036 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 3860037 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'testNodeAdded_shard1_replica_n1' using configuration from collection testNodeAdded, trusted=true
[junit4] 2> 3860036 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'testNodeAdded_shard1_replica_n2' using configuration from collection testNodeAdded, trusted=true
[junit4] 2> 3860037 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38925.solr.core.testNodeAdded.shard1.replica_n2' (registry 'solr.core.testNodeAdded.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3860037 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38925.solr.core.testNodeAdded.shard1.replica_n1' (registry 'solr.core.testNodeAdded.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3860037 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3860037 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3860037 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.SolrCore [[testNodeAdded_shard1_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/tempDir-001/node1/testNodeAdded_shard1_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/tempDir-001/node1/./testNodeAdded_shard1_replica_n2/data/]
[junit4] 2> 3860037 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.c.SolrCore [[testNodeAdded_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/tempDir-001/node1/testNodeAdded_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/tempDir-001/node1/./testNodeAdded_shard1_replica_n1/data/]
[junit4] 2> 3860085 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3860085 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3860085 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3860085 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3860100 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3860100 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3860100 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3860100 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3860102 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7f36169f[testNodeAdded_shard1_replica_n1] main]
[junit4] 2> 3860102 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@7bc1814[testNodeAdded_shard1_replica_n2] main]
[junit4] 2> 3860103 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3860104 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 3860104 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3860104 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 3860105 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3860105 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3860105 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1592885005094223872
[junit4] 2> 3860105 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1592885005094223872
[junit4] 2> 3860107 INFO (searcherExecutor-12746-thread-1-processing-n:127.0.0.1:38925_solr x:testNodeAdded_shard1_replica_n1 s:shard1 c:testNodeAdded r:core_node3) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.c.SolrCore [testNodeAdded_shard1_replica_n1] Registered new searcher Searcher@7f36169f[testNodeAdded_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3860108 INFO (searcherExecutor-12747-thread-1-processing-n:127.0.0.1:38925_solr x:testNodeAdded_shard1_replica_n2 s:shard1 c:testNodeAdded r:core_node4) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.SolrCore [testNodeAdded_shard1_replica_n2] Registered new searcher Searcher@7bc1814[testNodeAdded_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3860111 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.ZkShardTerms Failed to save terms, version is not match, retrying
[junit4] 2> 3860119 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3860119 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3860119 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded"} current state version: 0
[junit4] 2> 3860119 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:38925/solr/testNodeAdded_shard1_replica_n2/
[junit4] 2> 3860120 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=testNodeAdded_shard1_replica_n2 url=http://127.0.0.1:38925/solr START replicas=[http://127.0.0.1:38925/solr/testNodeAdded_shard1_replica_n1/] nUpdates=100
[junit4] 2> 3860122 INFO (qtp2092840606-33576) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.c.S.Request [testNodeAdded_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3860220 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testNodeAdded/state.json version: 2
[junit4] 2> 3860221 INFO (zkCallback-6380-thread-2-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3860221 INFO (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3860420 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=testNodeAdded_shard1_replica_n2 url=http://127.0.0.1:38925/solr DONE. We have no versions. sync failed.
[junit4] 2> 3860420 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 3860420 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 3860420 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3860427 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:38925/solr/testNodeAdded_shard1_replica_n2/ shard1
[junit4] 2> 3860429 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "state":"active"} current state version: 0
[junit4] 2> 3860535 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testNodeAdded/state.json version: 3
[junit4] 2> 3860536 INFO (zkCallback-6380-thread-2-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3860536 INFO (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3860584 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3860587 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "core_node_name":"core_node4",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"} current state version: 0
[junit4] 2> 3860587 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node4 x:testNodeAdded_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node4&collection.configName=conf&newCollection=true&name=testNodeAdded_shard1_replica_n2&action=CREATE&numShards=1&collection=testNodeAdded&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1598
[junit4] 2> 3860587 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "core_node_name":"core_node4",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"}
[junit4] 2> 3860588 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node4:{"core":"testNodeAdded_shard1_replica_n2","base_url":"http://127.0.0.1:38925/solr","node_name":"127.0.0.1:38925_solr","state":"active","type":"NRT","leader":"true"}
[junit4] 2> 3860588 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(testNodeAdded//collections/testNodeAdded/state.json/4)={
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"down",
[junit4] 2> "type":"NRT"},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "leader":"true"}}}}}
[junit4] 2> 3860675 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3860675 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 3860695 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testNodeAdded/state.json version: 4
[junit4] 2> 3860696 INFO (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3860696 INFO (zkCallback-6380-thread-2-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3861122 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "core_node_name":"core_node3",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"} current state version: 0
[junit4] 2> 3861122 INFO (qtp2092840606-33572) [n:127.0.0.1:38925_solr c:testNodeAdded s:shard1 r:core_node3 x:testNodeAdded_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=testNodeAdded_shard1_replica_n1&action=CREATE&numShards=1&collection=testNodeAdded&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2134
[junit4] 2> 3861122 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "core_node_name":"core_node3",
[junit4] 2> "roles":null,
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "numShards":"1",
[junit4] 2> "state":"active",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded",
[junit4] 2> "type":"NRT",
[junit4] 2> "operation":"state"}
[junit4] 2> 3861122 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Will update state for replica: core_node3:{"core":"testNodeAdded_shard1_replica_n1","base_url":"http://127.0.0.1:38925/solr","node_name":"127.0.0.1:38925_solr","state":"active","type":"NRT"}
[junit4] 2> 3861122 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ReplicaMutator Collection is now: DocCollection(testNodeAdded//collections/testNodeAdded/state.json/5)={
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT"},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "leader":"true"}}}}}
[junit4] 2> 3861125 INFO (qtp2092840606-33571) [n:127.0.0.1:38925_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3861223 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/testNodeAdded/state.json version: 5
[junit4] 2> 3861224 INFO (zkCallback-6380-thread-2-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3861224 INFO (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/testNodeAdded/state.json] for collection [testNodeAdded] has occurred - updating... (live nodes size: [1])
[junit4] 2> 3861676 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3861676 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 1
[junit4] 2> 3862126 INFO (qtp2092840606-33571) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&name=testNodeAdded&nrtReplicas=2&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=3462
[junit4] 2> 3862131 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.testNodeAdded.shard1.replica_n1:INDEX.sizeInBytes&key=solr.core.testNodeAdded.shard1.replica_n2:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 3862133 INFO (qtp2092840606-33576) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3862134 DEBUG (qtp2092840606-33574) [n:127.0.0.1:38925_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3862134 INFO (qtp2092840606-33574) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=5
[junit4] 2> 3862135 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
[junit4] 2> 3862135 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.Server jetty-9.4.8.v20171121, build timestamp: 2017-11-22T10:27:37+13:00, git hash: 82b8fb23f757335bb3329d540ce37a2a2615f0a8
[junit4] 2> 3862136 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:38925_solr]
[junit4] 2> 3862136 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=1, actions=[{name=compute_plan, class=solr.ComputePlanAction}, {name=test, class=org.apache.solr.cloud.autoscaling.ComputePlanActionTest$AssertingTriggerAction}], enabled=true}
[junit4] 2> 3862136 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
[junit4] 2> 3862136 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3862137 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.session DefaultSessionIdManager workerName=node0
[junit4] 2> 3862137 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
[junit4] 2> 3862137 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.session No SessionScavenger set, using defaults
[junit4] 2> 3862137 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.session Scavenging every 600000ms
[junit4] 2> 3862137 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@26ace357{/solr,null,AVAILABLE}
[junit4] 2> 3862138 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3a319d8e{HTTP/1.1,[http/1.1]}{127.0.0.1:55048}
[junit4] 2> 3862138 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.Server Started @3869426ms
[junit4] 2> 3862138 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=55048}
[junit4] 2> 3862138 ERROR (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3862138 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 8.0.0
[junit4] 2> 3862138 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3862138 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3862138 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2018-02-20T02:25:20.380Z
[junit4] 2> 3862140 INFO (zkConnectionManagerCallback-6392-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3862141 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3862148 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:61117/solr
[junit4] 2> 3862150 INFO (zkConnectionManagerCallback-6396-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3862153 INFO (zkConnectionManagerCallback-6398-thread-1-processing-n:127.0.0.1:55048_solr) [n:127.0.0.1:55048_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3862163 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [n:127.0.0.1:55048_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3862167 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [n:127.0.0.1:55048_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3862170 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [n:127.0.0.1:55048_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55048_solr
[junit4] 2> 3862171 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:55048_solr"} current state version: 0
[junit4] 2> 3862171 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:55048_solr
[junit4] 2> 3862172 INFO (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3862173 INFO (zkCallback-6388-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3862173 INFO (zkCallback-6388-thread-2) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3862173 INFO (zkCallback-6397-thread-1-processing-n:127.0.0.1:55048_solr) [n:127.0.0.1:55048_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3862239 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [n:127.0.0.1:55048_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55048.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3862246 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [n:127.0.0.1:55048_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55048.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3862247 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [n:127.0.0.1:55048_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_55048.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@68ced282
[junit4] 2> 3862248 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [n:127.0.0.1:55048_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-master-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.ComputePlanActionTest_4324EF2CAA18D31F-001/tempDir-001/node2/.
[junit4] 2> 3862677 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3862677 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 3862677 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:55048_solr at time 2610752371691143
[junit4] 2> 3862678 INFO (OverseerCollectionConfigSetProcessor-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3863679 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3863679 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 3863679 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger firing registered processor for nodes: [127.0.0.1:55048_solr] added at times [2610752371691143], now=2610753373949473
[junit4] 2> 3863680 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"9467727b60e87T3r6j706tidzw0b8wjy2i0cs3i",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":2610752371691143,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2610752371691143],
[junit4] 2> "nodeNames":["127.0.0.1:55048_solr"]}}
[junit4] 2> 3863680 DEBUG (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.ScheduledTriggers Pausing all triggers: [node_added_trigger]
[junit4] 2> 3863681 INFO (ScheduledTrigger-12741-thread-2) [n:127.0.0.1:38925_solr ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
[junit4] 2> 3863682 DEBUG (AutoscalingActionExecutor-12742-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"9467727b60e87T3r6j706tidzw0b8wjy2i0cs3i",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":2610752371691143,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2610752371691143],
[junit4] 2> "_enqueue_time_":2610753374627653,
[junit4] 2> "nodeNames":["127.0.0.1:55048_solr"]}}
[junit4] 2> 3863682 DEBUG (AutoscalingActionExecutor-12742-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.ComputePlanAction -- processing event: {
[junit4] 2> "id":"9467727b60e87T3r6j706tidzw0b8wjy2i0cs3i",
[junit4] 2> "source":"node_added_trigger",
[junit4] 2> "eventTime":2610752371691143,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2610752371691143],
[junit4] 2> "_enqueue_time_":2610753374627653,
[junit4] 2> "nodeNames":["127.0.0.1:55048_solr"]}} with context properties: {BEFORE_ACTION=[compute_plan]}
[junit4] 2> 3863683 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={wt=javabin&version=2&key=solr.core.testNodeAdded.shard1.replica_n1:INDEX.sizeInBytes&key=solr.core.testNodeAdded.shard1.replica_n2:INDEX.sizeInBytes} status=0 QTime=0
[junit4] 2> 3863685 INFO (qtp2092840606-33574) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3863688 INFO (qtp365583514-33623) [n:127.0.0.1:55048_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3863689 INFO (AutoscalingActionExecutor-12742-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.ComputePlanAction Computed Plan: action=MOVEREPLICA&collection=testNodeAdded&targetNode=127.0.0.1:55048_solr&inPlaceMove=true&replica=core_node3
[junit4] 2> 3863689 INFO (AutoscalingActionExecutor-12742-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.s.c.a.PolicyHelper returnSession, curr-time 2610753383 sessionWrapper.createTime 2610753383062541, this.sessionWrapper.createTime 2610753383062541
[junit4] 2> 3863689 DEBUG (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.a.ComputePlanActionTest -------------_ FINAL STATE --------------
[junit4] 2> 3863691 DEBUG (AutoscalingActionExecutor-12742-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.ScheduledTriggers Resuming trigger: node_added_trigger after 5000ms
[junit4] 2> 3863691 DEBUG (AutoscalingActionExecutor-12742-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.ScheduledTriggers -- processing took 9 ms for event id=9467727b60e87T3r6j706tidzw0b8wjy2i0cs3i
[junit4] 2> 3863691 INFO (qtp2092840606-33575) [n:127.0.0.1:38925_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3863692 DEBUG (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.a.ComputePlanActionTest * Node values: 127.0.0.1:38925_solr
[junit4] 2> {
[junit4] 2> "node":"127.0.0.1:38925_solr",
[junit4] 2> "cores":2,
[junit4] 2> "port":"38925",
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "ip_1":"1",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"127",
[junit4] 2> "freedisk":12.304348945617676}
[junit4] 2> 3863694 INFO (qtp365583514-33627) [n:127.0.0.1:55048_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CONTAINER.fs.usableSpace,CORE.coreName&wt=javabin&version=2&group=solr.node,solr.core} status=0 QTime=0
[junit4] 2> 3863694 DEBUG (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.a.ComputePlanActionTest * Node values: 127.0.0.1:55048_solr
[junit4] 2> {
[junit4] 2> "node":"127.0.0.1:55048_solr",
[junit4] 2> "cores":0,
[junit4] 2> "port":"55048",
[junit4] 2> "host":"127.0.0.1",
[junit4] 2> "ip_1":"1",
[junit4] 2> "ip_3":"0",
[junit4] 2> "ip_2":"0",
[junit4] 2> "ip_4":"127",
[junit4] 2> "freedisk":12.304348945617676}
[junit4] 2> 3863694 DEBUG (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.a.ComputePlanActionTest * Live nodes: [127.0.0.1:38925_solr, 127.0.0.1:55048_solr]
[junit4] 2> 3863695 DEBUG (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.a.ComputePlanActionTest * Collection testNodeAdded state: DocCollection(testNodeAdded//collections/testNodeAdded/state.json/6)={
[junit4] 2> "pullReplicas":"0",
[junit4] 2> "replicationFactor":"2",
[junit4] 2> "shards":{"shard1":{
[junit4] 2> "range":"80000000-7fffffff",
[junit4] 2> "state":"active",
[junit4] 2> "replicas":{
[junit4] 2> "core_node3":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n1",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT"},
[junit4] 2> "core_node4":{
[junit4] 2> "core":"testNodeAdded_shard1_replica_n2",
[junit4] 2> "base_url":"http://127.0.0.1:38925/solr",
[junit4] 2> "node_name":"127.0.0.1:38925_solr",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "leader":"true"}}}},
[junit4] 2> "router":{"name":"compositeId"},
[junit4] 2> "maxShardsPerNode":"1",
[junit4] 2> "autoAddReplicas":"false",
[junit4] 2> "nrtReplicas":"2",
[junit4] 2> "tlogReplicas":"0"}
[junit4] 2> 3863695 INFO (TEST-ComputePlanActionTest.testNodeAdded-seed#[4324EF2CAA18D31F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAdded
[junit4] 2> 3863713 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLost
[junit4] 2> 3863714 DEBUG (zkCallback-6380-thread-1-processing-n:127.0.0.1:38925_solr) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
[junit4] 2> 3863714 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
[junit4] 2> 3863715 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@15eebe3c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3863716 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1096261037
[junit4] 2> 3863716 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3863716 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3863717 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@13ef9a89: rootName = solr_38925, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@55cce09d
[junit4] 2> 3863717 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3863717 DEBUG (OverseerAutoScalingTriggerThread-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
[junit4] 2> 3863721 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3863721 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@1e29036: rootName = solr_38925, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@55fbcd57
[junit4] 2> 3863725 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3863725 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6f1e7c5c: rootName = solr_38925, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@483bdb63
[junit4] 2> 3863728 INFO (TEST-ComputePlanActionTest.testNodeLost-seed#[4324EF2CAA18D31F]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:38925_solr
[junit4] 2> 3863729 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0.1:38925_solr ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"downnode",
[junit4] 2> "node_name":"127.0.0.1:38925_solr"} current state version: 0
[junit4] 2> 3863729 DEBUG (OverseerStateUpdate-72228692014661635-127.0.0.1:38925_solr-n_0000000000) [n:127.0.0
[...truncated too long message...]
eper.stopScavenging Stopped scavenging
[junit4] 2> 49313 T12 oejs.AbstractConnector.doStop Stopped ServerConnector@4fa39e5e{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 49313 T12 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1334190035
[junit4] 2> 49313 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 49314 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 49314 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 49319 T204 x:x:collection1 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@5a987012
[junit4] 2> 49320 T204 x:x:collection1 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.core.collection1, tag=1519939602
[junit4] 2> 49326 T12 oejsh.ContextHandler.doStop Stopped o.e.j.s.ServletContextHandler@55afd82e{/,null,UNAVAILABLE}
[junit4] 2> 49326 T12 oejss.HouseKeeper.stopScavenging Stopped scavenging
[junit4] 2> 49332 T12 oejs.AbstractConnector.doStop Stopped ServerConnector@74a8fa9c{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 49333 T12 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1596671301
[junit4] 2> 49334 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 49334 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 49334 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 49336 T206 x:x:collection1 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@352e7052
[junit4] 2> 49337 T206 x:x:collection1 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.core.collection1, tag=892235858
[junit4] 2> 49346 T12 oejsh.ContextHandler.doStop Stopped o.e.j.s.ServletContextHandler@4edd8a5b{/,null,UNAVAILABLE}
[junit4] 2> 49346 T12 oejss.HouseKeeper.stopScavenging Stopped scavenging
[junit4] 2> 49353 T12 oejs.AbstractConnector.doStop Stopped ServerConnector@24261373{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 49354 T12 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=2093603989
[junit4] 2> 49354 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 49355 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 49355 T12 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 49357 T208 x:x:collection1 oasc.SolrCore.close [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@2f1a2630
[junit4] 2> 49357 T208 x:x:collection1 oasm.SolrMetricManager.closeReporters Closing metric reporters for registry=solr.core.collection1, tag=790242864
[junit4] 2> 49362 T12 oejsh.ContextHandler.doStop Stopped o.e.j.s.ServletContextHandler@66977b75{/,null,UNAVAILABLE}
[junit4] 2> 49363 T12 oejss.HouseKeeper.stopScavenging Stopped scavenging
[junit4] 2> 49373 T12 oas.SolrTestCaseJ4.clearObjectTrackerAndCheckEmpty ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Feb 20, 2018 2:48:14 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
[junit4] 2> Feb 20, 2018 2:48:34 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 1 thread leaked from SUITE scope at org.apache.solr.handler.clustering.DistributedClusteringComponentTest:
[junit4] 2> 1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Feb 20, 2018 2:48:34 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest]
[junit4] 2> Feb 20, 2018 2:48:37 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> SEVERE: There are still zombie threads that couldn't be terminated:
[junit4] 2> 1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] 2> at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {snippet=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), id=FSTOrd50, text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), title=FST50, url=FSTOrd50}, docValues:{}, maxPointsInLeafNode=1083, maxMBSortInHeap=6.1275688775296855, sim=Asserting(org.apache.lucene.search.similarities.AssertingSimilarity@4227ee9a), locale=ja, timezone=Asia/Ujung_Pandang
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=2,free=95763488,total=188743680
[junit4] 2> NOTE: All tests run in this JVM: [DistributedClusteringComponentTest]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=DistributedClusteringComponentTest -Dtests.seed=8AEF5CD4F3FBC58 -Dtests.slow=true -Dtests.locale=ja -Dtests.timezone=Asia/Ujung_Pandang -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | DistributedClusteringComponentTest (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.handler.clustering.DistributedClusteringComponentTest:
[junit4] > 1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([8AEF5CD4F3FBC58]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=37, name=qtp1278777971-37, state=TIMED_WAITING, group=TGRP-DistributedClusteringComponentTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.reservedWait(ReservedThreadExecutor.java:308)
[junit4] > at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:373)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:708)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([8AEF5CD4F3FBC58]:0)
[junit4] Completed [3/3 (1!)] on J0 in 73.12s, 1 test, 2 errors <<< FAILURES!
[...truncated 44710 lines...]