You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2017/12/24 13:07:10 UTC
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 349 -
Still Unstable!
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/349/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
2 tests failed.
FAILED: org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnly
Error Message:
Unexpected number of elements in the group for intGSF: 5
Stack Trace:
java.lang.AssertionError: Unexpected number of elements in the group for intGSF: 5
at __randomizedtesting.SeedInfo.seed([F67057F252F1943C:6DCB39AA1FA9A662]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.apache.solr.cloud.DocValuesNotIndexedTest.testGroupingDVOnly(DocValuesNotIndexedTest.java:377)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=9759, name=jetty-launcher-2031-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 2) Thread[id=9762, name=jetty-launcher-2031-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
1) Thread[id=9759, name=jetty-launcher-2031-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
2) Thread[id=9762, name=jetty-launcher-2031-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
at __randomizedtesting.SeedInfo.seed([F67057F252F1943C]:0)
Build Log:
[...truncated 12251 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_F67057F252F1943C-001/init-core-data-001
[junit4] 2> 1327267 WARN (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=1 numCloses=1
[junit4] 2> 1327267 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 1327268 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 1327268 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 1327293 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_F67057F252F1943C-001/tempDir-001
[junit4] 2> 1327293 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1327293 INFO (Thread-2464) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1327293 INFO (Thread-2464) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1327295 ERROR (Thread-2464) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1327393 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:64367
[junit4] 2> 1327395 INFO (zkConnectionManagerCallback-2034-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327400 INFO (jetty-launcher-2031-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1327400 INFO (jetty-launcher-2031-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 1327402 INFO (jetty-launcher-2031-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@36b26a5{/solr,null,AVAILABLE}
[junit4] 2> 1327403 INFO (jetty-launcher-2031-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4ead71de{/solr,null,AVAILABLE}
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@75dbb8d3{HTTP/1.1,[http/1.1]}{127.0.0.1:60630}
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@a6bbb7a{HTTP/1.1,[http/1.1]}{127.0.0.1:45980}
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-1) [ ] o.e.j.s.Server Started @1330216ms
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-2) [ ] o.e.j.s.Server Started @1330216ms
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60630}
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=45980}
[junit4] 2> 1327405 ERROR (jetty-launcher-2031-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1327405 ERROR (jetty-launcher-2031-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 1327406 INFO (jetty-launcher-2031-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 1327405 INFO (jetty-launcher-2031-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1327406 INFO (jetty-launcher-2031-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 1327406 INFO (jetty-launcher-2031-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-24T11:53:02.157Z
[junit4] 2> 1327406 INFO (jetty-launcher-2031-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-24T11:53:02.157Z
[junit4] 2> 1327408 INFO (zkConnectionManagerCallback-2037-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327408 INFO (zkConnectionManagerCallback-2038-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327408 INFO (jetty-launcher-2031-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1327408 INFO (jetty-launcher-2031-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 1327415 INFO (jetty-launcher-2031-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64367/solr
[junit4] 2> 1327415 INFO (jetty-launcher-2031-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:64367/solr
[junit4] 2> 1327417 INFO (zkConnectionManagerCallback-2046-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327417 INFO (zkConnectionManagerCallback-2044-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327419 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327419 INFO (zkConnectionManagerCallback-2050-thread-1-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327548 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1327549 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1327549 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:45980_solr
[junit4] 2> 1327550 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.Overseer Overseer (id=99229131213701126-127.0.0.1:45980_solr-n_0000000000) starting
[junit4] 2> 1327550 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60630_solr
[junit4] 2> 1327552 INFO (zkCallback-2047-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1327553 INFO (zkCallback-2049-thread-1-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1327564 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45980_solr
[junit4] 2> 1327567 INFO (zkCallback-2049-thread-1-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1327569 INFO (zkCallback-2047-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1327654 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 1327655 WARN (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 1327655 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 1327656 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 1327658 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 1327659 WARN (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 1327659 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 1327660 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 1327673 INFO (jetty-launcher-2031-thread-1-EventThread) [n:127.0.0.1:60630_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 1327673 INFO (jetty-launcher-2031-thread-2-EventThread) [n:127.0.0.1:45980_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 1327696 WARN (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 1327700 WARN (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 1327743 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45980.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23f8d3a6
[junit4] 2> 1327743 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60630.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23f8d3a6
[junit4] 2> 1327749 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45980.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23f8d3a6
[junit4] 2> 1327749 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_45980.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23f8d3a6
[junit4] 2> 1327749 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60630.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23f8d3a6
[junit4] 2> 1327749 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60630.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23f8d3a6
[junit4] 2> 1327750 INFO (jetty-launcher-2031-thread-2) [n:127.0.0.1:45980_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_F67057F252F1943C-001/tempDir-001/node2/.
[junit4] 2> 1327750 INFO (jetty-launcher-2031-thread-1) [n:127.0.0.1:60630_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_F67057F252F1943C-001/tempDir-001/node1/.
[junit4] 2> 1327781 INFO (zkConnectionManagerCallback-2056-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1327813 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
[junit4] 2> 1327850 INFO (qtp1954351823-9717) [n:127.0.0.1:45980_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1327850 INFO (qtp1954351823-9717) [n:127.0.0.1:45980_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1327851 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=test&e=1514152382568&s=20NuCm1aLPdzS5ltb2gIuV7ffQw="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:02 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:02 GMT
[junit4] 2> 1327851 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
[junit4] 2> 1327883 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
[junit4] 2> 1327883 INFO (qtp1954351823-9718) [ ] o.a.s.s.SolrDispatchFilter Error authenticating
[junit4] 2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1327884 WARN (qtp1954351823-9718) [ ] o.e.j.s.ServletHandler /solr/admin/collections
[junit4] 2> org.apache.solr.common.SolrException: Error during request authentication,
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> ... 26 more
[junit4] 2> 1327885 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
[junit4] 2> 1327920 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
[junit4] 2> 1327945 WARN (qtp1954351823-9721) [ ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
[junit4] 2> id: invalid user name: "bar"
[junit4] 2>
[junit4] 2> 1327951 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=test&e=1514152382673&s=LLhRGW+PaIG9oGuuCpkXJsTNzYo="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:02 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:02 GMT
[junit4] 2> 1327951 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
[junit4] 2> 1327984 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
[junit4] 2> 1327989 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=test&e=1514152382736&s=W+EMdnJ7Z2gYe/Jbyb8/TewYMcw="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:02 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:02 GMT
[junit4] 2> 1327989 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
[junit4] 2> 1328011 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
[junit4] 2> 1333058 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=test&e=1514152382763&s=daogwdjcB22zbtGvR4dBYsD88DA="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:02 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:02 GMT
[junit4] 2> 1333059 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
[junit4] 2> 1333104 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
[junit4] 2> 1333107 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=test&e=1514152387856&s=j6LSznhVZS9y0IAVbBZTINonKjg="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:07 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:07 GMT
[junit4] 2> 1333108 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
[junit4] 2> 1333149 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
[junit4] 2> 1349774 WARN (jetty-launcher-2031-thread-2-SendThread(127.0.0.1:64367)) [n:127.0.0.1:45980_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 16717ms for sessionid 0x160885e7d5d0006
[junit4] 2> 1349775 WARN (jetty-launcher-2031-thread-1-SendThread(127.0.0.1:64367)) [n:127.0.0.1:60630_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 22024ms for sessionid 0x160885e7d5d0005
[junit4] 2> 1349776 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160885e7d5d0006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1349776 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1514152387900&s=a+YSxb2gzUHscFpjSbxMRs/ArmU="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:07 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:07 GMT
[junit4] 2> 1349776 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
[junit4] 2> 1349817 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
[junit4] 2> 1349819 INFO (zkConnectionManagerCallback-2059-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1349878 INFO (jetty-launcher-2031-thread-2-EventThread) [n:127.0.0.1:45980_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 1349885 WARN (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5f06e356 name: ZooKeeperConnection Watcher:127.0.0.1:64367/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 1349885 WARN (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 1349886 WARN (zkConnectionManagerCallback-2050-thread-1-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2cc0a5f name: ZooKeeperConnection Watcher:127.0.0.1:64367/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 1349886 WARN (zkConnectionManagerCallback-2050-thread-1-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 1350812 INFO (zkConnectionManagerCallback-2062-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1350814 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1350816 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:64367/solr ready
[junit4] 2> 1350817 INFO (qtp1954351823-9719) [n:127.0.0.1:45980_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1351005 WARN (jetty-launcher-2031-thread-1-SendThread(127.0.0.1:64367)) [n:127.0.0.1:60630_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x160885e7d5d0007 has expired
[junit4] 2> 1351056 WARN (jetty-launcher-2031-thread-1-SendThread(127.0.0.1:64367)) [n:127.0.0.1:60630_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x160885e7d5d0005 has expired
[junit4] 2> 1351056 WARN (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5f06e356 name: ZooKeeperConnection Watcher:127.0.0.1:64367/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 1351056 WARN (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 1351056 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1351056 WARN (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 1351060 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr-EventThread) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1351060 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 1351061 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 1351065 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 1351081 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60630_solr
[junit4] 2> 1351083 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 1351083 INFO (zkConnectionManagerCallback-2048-thread-1-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 1351084 INFO (zkCallback-2047-thread-4-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1351086 INFO (zkCallback-2061-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1351244 INFO (zkConnectionManagerCallback-2050-thread-1-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 1351381 INFO (OverseerThreadFactory-3917-thread-1-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
[junit4] 2> 1351502 INFO (OverseerStateUpdate-99229131213701126-127.0.0.1:45980_solr-n_0000000000) [n:127.0.0.1:45980_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"forwardingCollection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"forwardingCollection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:60630/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 1351713 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 1351741 WARN (jetty-launcher-2031-thread-2-SendThread(127.0.0.1:64367)) [n:127.0.0.1:45980_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x160885e7d5d0008 has expired
[junit4] 2> 1351825 INFO (zkCallback-2047-thread-3-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1351825 INFO (zkCallback-2047-thread-4-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1352753 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 1352796 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica_n1] Schema name=test
[junit4] 2> 1352859 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 1352888 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica_n1' using configuration from collection forwardingCollection, trusted=true
[junit4] 2> 1352888 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60630.solr.core.forwardingCollection.shard1.replica_n1' (registry 'solr.core.forwardingCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@23f8d3a6
[junit4] 2> 1352888 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 1352888 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_F67057F252F1943C-001/tempDir-001/node1/forwardingCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_F67057F252F1943C-001/tempDir-001/node1/./forwardingCollection_shard1_replica_n1/data/]
[junit4] 2> 1352890 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=11.0302734375, floorSegmentMB=0.91015625, forceMergeDeletesPctAllowed=18.24985939423263, segmentsPerTier=43.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
[junit4] 2> 1352937 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 1352937 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 1352938 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 1352938 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 1352939 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=44, maxMergeAtOnceExplicit=47, maxMergedSegmentMB=67.1640625, floorSegmentMB=0.5390625, forceMergeDeletesPctAllowed=3.830005898331051, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
[junit4] 2> 1352939 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@1ebd8271[forwardingCollection_shard1_replica_n1] main]
[junit4] 2> 1352941 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 1352945 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 1352960 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 1352960 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 1352960 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587666126331969536
[junit4] 2> 1352961 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 1352961 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 1352961 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 1352961 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 1352961 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 1352962 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 1352963 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 1352964 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 1352965 INFO (searcherExecutor-3922-thread-1-processing-n:127.0.0.1:60630_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] Registered new searcher Searcher@1ebd8271[forwardingCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 1352967 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 1352967 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 1352967 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:60630/solr/forwardingCollection_shard1_replica_n1/
[junit4] 2> 1352967 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 1352967 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:60630/solr/forwardingCollection_shard1_replica_n1/ has no replicas
[junit4] 2> 1352967 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 1352971 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:60630/solr/forwardingCollection_shard1_replica_n1/ shard1
[junit4] 2> 1353075 INFO (zkCallback-2047-thread-3-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1353075 INFO (zkCallback-2047-thread-4-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1353145 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 1353148 INFO (qtp285414748-9710) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1436
[junit4] 2> 1353149 WARN (httpShardExecutor-3911-thread-1-processing-n:127.0.0.1:45980_solr [http:////127.0.0.1:60630//solr] http:////127.0.0.1:60630//solr) [n:127.0.0.1:45980_solr ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1514152406463&s=C9CX7/YoKQ1Jmw7j5G2POmfxorU="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:26 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:26 GMT
[junit4] 2> 1353154 INFO (qtp1954351823-9719) [n:127.0.0.1:45980_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 1353251 INFO (zkCallback-2047-thread-3-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1353251 INFO (zkCallback-2047-thread-4-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 1353385 INFO (OverseerCollectionConfigSetProcessor-99229131213701126-127.0.0.1:45980_solr-n_0000000000) [n:127.0.0.1:45980_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 1354146 ERROR (jetty-launcher-2031-thread-1-EventThread) [n:127.0.0.1:60630_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1212)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:302)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl$4.call(GetDataBuilderImpl.java:291)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:107)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 1354166 INFO (qtp1954351823-9719) [n:127.0.0.1:45980_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=3350
[junit4] 2> 1354167 WARN (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1514152405568&s=9kuIY4e056kDnzJHun5aPBnf5bc="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:25 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:25 GMT
[junit4] 2> 1354167 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
[junit4] 2> 1354168 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
[junit4] 2> 1354172 INFO (qtp1954351823-9721) [n:127.0.0.1:45980_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 1354179 INFO (qtp285414748-9712) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1354179 WARN (qtp1954351823-9721) [n:127.0.0.1:45980_solr ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1514152408927&s=QcByL7DI3CRLEnmI5uQDFVa6ets="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:28 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:28 GMT
[junit4] 2> 1354186 WARN (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1514152408923&s=ZK38Sjy6f06NjXgL34RxCMAYtyU="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:28 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:28 GMT
[junit4] 2> 1354186 WARN (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=$&p=$&t=test&e=1514152408927&s=QcByL7DI3CRLEnmI5uQDFVa6ets="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:28 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:28 GMT
[junit4] 2> 1354191 INFO (qtp285414748-9715) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 1354191 WARN (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=user&p=user&t=test&e=1514152408941&s=DJPLp9g4DN1ZFDEq467WUyvG1nA="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:28 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:28 GMT
[junit4] 2> 1354192 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
[junit4] 2> 1354260 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
[junit4] 2> 1354266 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1514152409012&s=26m39V6V1QjHXkg8hdAwQ634rcg="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:29 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:29 GMT
[junit4] 2> 1354266 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
[junit4] 2> 1354302 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
[junit4] 2> 1354303 INFO (qtp1954351823-9718) [n:127.0.0.1:45980_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1354303 INFO (qtp1954351823-9718) [n:127.0.0.1:45980_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1354304 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=test&e=1514152409053&s=LuT3l8o01GLP8fYUrlYwl3BSTIU="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:29 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:29 GMT
[junit4] 2> 1354304 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
[junit4] 2> 1354335 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
[junit4] 2> 1354336 INFO (qtp1954351823-9720) [n:127.0.0.1:45980_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 1354337 INFO (qtp1954351823-9720) [n:127.0.0.1:45980_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 1354337 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=test&e=1514152409087&s=LMpC2+ht9MmVgrIzfHETqgvnTJc="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:29 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:29 GMT
[junit4] 2> 1354337 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
[junit4] 2> 1354375 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
[junit4] 2> 1354377 WARN (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[F67057F252F1943C]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=test&e=1514152409126&s=AioQykK7OmZCapV0rgchD4jV1ZY="; Path=/; Domain=127.0.0.1; Expires=ned, 24-pro-2017 21:53:29 GMT; HttpOnly". Invalid 'expires' attribute: ned, 24-pro-2017 21:53:29 GMT
[junit4] 2> 1354377 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[F67057F252F1943C]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
[junit4] 2> 1354379 INFO (jetty-closer-2032-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@75dbb8d3{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1354379 INFO (jetty-closer-2032-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@a6bbb7a{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 1354380 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2011581309
[junit4] 2> 1354381 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=2081013437
[junit4] 2> 1354381 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1354381 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@327e762f: rootName = solr_60630, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@47e59738
[junit4] 2> 1354381 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 1354381 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@12f452cd: rootName = solr_45980, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6dc0f274
[junit4] 2> 1354389 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 1354389 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7243cace: rootName = solr_60630, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1971d86c
[junit4] 2> 1354389 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 1354389 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@76bdc3a3: rootName = solr_45980, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1971d86c
[junit4] 2> 1354394 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 1354394 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 1354394 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3bf79da8: rootName = solr_45980, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@1647dfc9
[junit4] 2> 1354394 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6413ac6d: rootName = solr_60630, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@1647dfc9
[junit4] 2> 1354396 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:60630_solr
[junit4] 2> 1354396 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:45980_solr
[junit4] 2> 1354397 INFO (jetty-closer-2032-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1354397 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 1354397 INFO (zkCallback-2047-thread-4-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 1354398 INFO (jetty-closer-2032-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99229131213701126-127.0.0.1:45980_solr-n_0000000000) closing
[junit4] 2> 1354398 INFO (OverseerStateUpdate-99229131213701126-127.0.0.1:45980_solr-n_0000000000) [n:127.0.0.1:45980_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:45980_solr
[junit4] 2> 1354398 INFO (zkCallback-2049-thread-4-processing-n:127.0.0.1:45980_solr) [n:127.0.0.1:45980_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (0)
[junit4] 2> 1354399 INFO (coreCloseExecutor-3928-thread-1) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@7ca9c882
[junit4] 2> 1354399 INFO (coreCloseExecutor-3928-thread-1) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.forwardingCollection.shard1.replica_n1, tag=2091501698
[junit4] 2> 1354399 INFO (coreCloseExecutor-3928-thread-1) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@5135481c: rootName = solr_60630, domain = solr.core.forwardingCollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.forwardingCollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@f80592a
[junit4] 2> 1354399 WARN (OverseerAutoScalingTriggerThread-99229131213701126-127.0.0.1:45980_solr-n_0000000000) [n:127.0.0.1:45980_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1354404 INFO (zkCallback-2047-thread-4-processing-n:127.0.0.1:60630_solr) [n:127.0.0.1:60630_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60630_solr
[junit4] 2> 1354404 ERROR (Thread[Thread-2483,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:45980_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 1354409 INFO (jetty-closer-2032-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@36b26a5{/solr,null,UNAVAILABLE}
[junit4] 2> 1354419 INFO (coreCloseExecutor-3928-thread-1) [n:127.0.0.1:60630_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.forwardingCollection.shard1.leader, tag=2091501698
[junit4] 2> 1354426 ERROR (Thread[Thread-2481,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:60630_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 1354430 INFO (jetty-closer-2032-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4ead71de{/solr,null,UNAVAILABLE}
[junit4] 2> 1354432 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1354432 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64367 64367
[junit4] 2> 1354998 INFO (Thread-2464) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:64367 64367
[junit4] 2> 1354999 WARN (Thread-2464) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2> 4 /solr/clusterprops.json
[junit4] 2> 3 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/collections/forwardingCollection/state.json
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/overseer_elect/election/99229131213701126-127.0.0.1:45980_solr-n_0000000000
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/collections
[junit4] 2> 2 /solr/live_nodes
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 1355027 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[F67057F252F1943C]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Dec 24, 2017 11:53:29 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 3 leaked thread(s).
[junit4] 2> Dec 24, 2017 11:53:49 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] 2> 1) Thread[id=9759, name=jetty-launcher-2031-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 2) Thread[id=9762, name=jetty-launcher-2031-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Dec 24, 2017 11:53:49 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=9759, name=jetty-launcher-2031-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> 2) Thread[id=9762, name=jetty-launcher-2031-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> 1375128 ERROR (jetty-launcher-2031-thread-2-EventThread) [n:127.0.0.1:45980_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.frame
[...truncated too long message...]
Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3117812 INFO (Thread-6113) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:63705 63705
[junit4] 2> 3117811 WARN (jetty-launcher-2031-thread-1-SendThread(127.0.0.1:64367)) [n:127.0.0.1:60630_solr ] o.a.z.ClientCnxn Session 0x0 for server null, unexpected error, closing socket connection and attempting reconnect
[junit4] 2> java.net.ConnectException: Connection refused
[junit4] 2> at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
[junit4] 2> at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:717)
[junit4] 2> at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:361)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1141)
[junit4] 2> 3117813 WARN (Thread-6113) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 6 /solr/configs/conf1/managed-schema
[junit4] 2> 4 /solr/configs/conf1
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 14 /solr/collections/dv_coll/state.json
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/configs/conf1/managed-schema
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.DocValuesNotIndexedTest_F67057F252F1943C-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=true): {}, locale=es-EC, timezone=Africa/Cairo
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=210962664,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [ResourceLoaderTest, TestConfigSetImmutable, LeaderFailureAfterFreshStartTest, ZkCLITest, NodeMutatorTest, HttpTriggerListenerTest, TestExactSharedStatsCache, TestDFRSimilarityFactory, SolrIndexMetricsTest, TestBinaryField, OverseerModifyCollectionTest, HdfsRecoverLeaseTest, TestFieldCacheWithThreads, HdfsBasicDistributedZk2Test, ZkControllerTest, JavabinLoaderTest, CollectionsAPISolrJTest, TestBulkSchemaAPI, ChaosMonkeyNothingIsSafeTest, TestHdfsCloudBackupRestore, TestAuthorizationFramework, TestJoin, UninvertDocValuesMergePolicyTest, TestLMDirichletSimilarityFactory, TestFilteredDocIdSet, ZkStateReaderTest, TestRequestStatusCollectionAPI, TestFiltering, NumberUtilsTest, TestDefaultStatsCache, TestLegacyTerms, TestBinaryResponseWriter, DistributedFacetPivotWhiteBoxTest, SolrCoreMetricManagerTest, TestReplicaProperties, UtilsToolTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, RecoveryZkTest, DirectUpdateHandlerOptimizeTest, TestSolrCoreProperties, XsltUpdateRequestHandlerTest, TestBulkSchemaConcurrent, TestFieldCacheSortRandom, HighlighterTest, SolrShardReporterTest, TestSearcherReuse, TestMissingGroups, DeleteShardTest, FacetPivotSmallTest, TestCoreAdminApis, SolrXmlInZkTest, TimeZoneUtilsTest, TestHighFrequencyDictionaryFactory, BlockJoinFacetDistribTest, AsyncCallRequestStatusResponseTest, TestTlogReplica, HdfsThreadLeakTest, SpatialHeatmapFacetsTest, OverseerTest, TestDynamicFieldCollectionResource, CollectionReloadTest, TestMacroExpander, SuggesterTest, TestSolrCloudWithDelegationTokens, CSVRequestHandlerTest, FastVectorHighlighterTest, HighlighterConfigTest, TestSizeLimitedDistributedMap, TestLocalFSCloudBackupRestore, TestQueryWrapperFilter, HdfsCollectionsAPIDistributedZkTest, HttpSolrCallGetCoreTest, DirectUpdateHandlerTest, TestNoOpRegenerator, SolrCmdDistributorTest, TestEmbeddedSolrServerSchemaAPI, TestTriggerIntegration, RAMDirectoryFactoryTest, TestUtils, TestUnifiedSolrHighlighter, OverriddenZkACLAndCredentialsProvidersTest, CreateCollectionCleanupTest, MetricsHandlerTest, TestFieldCacheReopen, TestSchemaSimilarityResource, TestSortByMinMaxFunction, UpdateLogTest, URLClassifyProcessorTest, DirectSolrConnectionTest, TestStressInPlaceUpdates, TestIntervalFaceting, TestTolerantUpdateProcessorRandomCloud, TestConfigSets, DefaultValueUpdateProcessorTest, TestEmbeddedSolrServerAdminHandler, ConfigureRecoveryStrategyTest, TestPHPSerializedResponseWriter, TestSolrCloudWithSecureImpersonation, TestPrepRecovery, ManagedSchemaRoundRobinCloudTest, ImplicitSnitchTest, JvmMetricsTest, SolrMetricReporterTest, TestHalfAndHalfDocValues, SolrInfoBeanTest, BinaryUpdateRequestHandlerTest, TestJsonFacetRefinement, TestLegacyNumericRangeQueryBuilder, WordBreakSolrSpellCheckerTest, TestRangeQuery, LeaderElectionTest, TestCloudDeleteByQuery, TestQuerySenderListener, ShardRoutingTest, IndexBasedSpellCheckerTest, AtomicUpdatesTest, TestRawTransformer, TestCustomSort, MetricUtilsTest, TestDistribStateManager, HdfsChaosMonkeyNothingIsSafeTest, SubstringBytesRefFilterTest, SpellCheckCollatorTest, TestNodeAddedTrigger, SolrRequestParserTest, TemplateUpdateProcessorTest, TestHighlightDedupGrouping, CdcrBidirectionalTest, TestJmxIntegration, ExecutePlanActionTest, TestUseDocValuesAsStored2, BasicDistributedZkTest, TestSolrCoreSnapshots, TestIndexingPerformance, AutoScalingHandlerTest, SearchRateTriggerTest, TestSegmentSorting, TestSimpleTextCodec, DistribJoinFromCollectionTest, TestSolrCLIRunExample, TestSQLHandler, DistributedSpellCheckComponentTest, SimpleMLTQParserTest, TestFieldTypeCollectionResource, TestSweetSpotSimilarityFactory, TestCSVResponseWriter, TestCursorMarkWithoutUniqueKey, DistributedVersionInfoTest, OutputWriterTest, LargeFieldTest, TestExpandComponent, TestCodecSupport, AddReplicaTest, TestRandomRequestDistribution, TestDeleteCollectionOnDownNodes, ConvertedLegacyTest, TestInPlaceUpdatesDistrib, FullHLLTest, RequiredFieldsTest, AddSchemaFieldsUpdateProcessorFactoryTest, BigEndianAscendingWordDeserializerTest, ConnectionReuseTest, DistribCursorPagingTest, TestHashQParserPlugin, DeleteInactiveReplicaTest, TestUpdate, TestCoreDiscovery, TestSolrDeletionPolicy2, HdfsDirectoryTest, PeerSyncReplicationTest, SearchHandlerTest, RemoteQueryErrorTest, TestSolr4Spatial, SharedFSAutoReplicaFailoverTest, TestRandomDVFaceting, HdfsUnloadDistributedZkTest, ScriptEngineTest, SyncSliceTest, SpellCheckComponentTest, TestSolrCloudWithHadoopAuthPlugin, TestLRUStatsCache, TestSolrConfigHandlerCloud, AnalysisErrorHandlingTest, TestFileDictionaryLookup, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, ClusterStateUpdateTest, ShardRoutingCustomTest, TestDistributedSearch, TestDistributedGrouping, TestRecovery, TermVectorComponentDistributedTest, TestRealTimeGet, TestStressReorder, TestReload, TestStressVersions, DistributedTermsComponentTest, StatsComponentTest, QueryElevationComponentTest, BadIndexSchemaTest, TestFunctionQuery, BasicFunctionalityTest, TestLazyCores, TestBadConfig, ShowFileRequestHandlerTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, CoreAdminHandlerTest, DocValuesTest, SuggesterTSTTest, TestStressLucene, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, SortByFunctionTest, TestRemoteStreaming, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, DebugComponentTest, TestSurroundQueryParser, LukeRequestHandlerTest, DisMaxRequestHandlerTest, TestQueryUtils, TestWriterPerf, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, TestOmitPositions, FileBasedSpellCheckerTest, TermsComponentTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, LoggingHandlerTest, RegexBoostProcessorTest, IndexSchemaTest, TestCollationField, ReturnFieldsTest, QueryParsingTest, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, TestComponentsName, TestLFUCache, AlternateDirectoryTest, TestQuerySenderNoQuery, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, MultiTermTest, TestDocSet, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, TestXIncludeConfig, EchoParamsTest, TestIBSimilarityFactory, TestFastWriter, TestFastOutputStream, OpenExchangeRatesOrgProviderTest, PluginInfoTest, DateMathParserTest, PreAnalyzedFieldTest, PrimUtilsTest, DateFieldTest, SpellingQueryConverterTest, DOMUtilTest, ClusterStateTest, TestSolrJ, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, FileUtilsTest, CircularListTest, TestRTGBase, CursorPagingTest, DistributedIntervalFacetingTest, SolrTestCaseJ4Test, TestTolerantSearch, AliasIntegrationTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CloudExitableDirectoryReaderTest, CollectionTooManyReplicasTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, DeleteLastCustomShardedReplicaTest, DocValuesNotIndexedTest]
[junit4] Completed [520/764 (2!)] on J0 in 10.76s, 4 tests, 1 failure <<< FAILURES!
[...truncated 49021 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 356 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/356/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testSetProperties
Error Message:
Stack Trace:
java.lang.AssertionError
at __randomizedtesting.SeedInfo.seed([98C4F0A068D35A9F:F3A027E9DBFECF1B]:0)
at org.junit.Assert.fail(Assert.java:92)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertTrue(Assert.java:54)
at org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.testSetProperties(TriggerIntegrationTest.java:1361)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 13580 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.TriggerIntegrationTest
[junit4] 2> 3946154 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.TriggerIntegrationTest_98C4F0A068D35A9F-001/init-core-data-001
[junit4] 2> 3946154 WARN (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=24 numCloses=24
[junit4] 2> 3946154 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3946156 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 3946156 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.TriggerIntegrationTest_98C4F0A068D35A9F-001/tempDir-001
[junit4] 2> 3946156 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3946156 INFO (Thread-8437) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3946156 INFO (Thread-8437) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3946159 ERROR (Thread-8437) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3946256 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:34588
[junit4] 2> 3946266 INFO (zkConnectionManagerCallback-4767-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946288 INFO (jetty-launcher-4764-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3946288 INFO (jetty-launcher-4764-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3946295 INFO (jetty-launcher-4764-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f0500d3{/solr,null,AVAILABLE}
[junit4] 2> 3946295 INFO (jetty-launcher-4764-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4036f898{/solr,null,AVAILABLE}
[junit4] 2> 3946296 INFO (jetty-launcher-4764-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4785b8ba{HTTP/1.1,[http/1.1]}{127.0.0.1:54064}
[junit4] 2> 3946296 INFO (jetty-launcher-4764-thread-2) [ ] o.e.j.s.Server Started @3950789ms
[junit4] 2> 3946296 INFO (jetty-launcher-4764-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=54064}
[junit4] 2> 3946296 INFO (jetty-launcher-4764-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@37d291f6{HTTP/1.1,[http/1.1]}{127.0.0.1:58429}
[junit4] 2> 3946296 ERROR (jetty-launcher-4764-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3946296 INFO (jetty-launcher-4764-thread-1) [ ] o.e.j.s.Server Started @3950790ms
[junit4] 2> 3946296 INFO (jetty-launcher-4764-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 3946297 INFO (jetty-launcher-4764-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=58429}
[junit4] 2> 3946297 INFO (jetty-launcher-4764-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3946297 INFO (jetty-launcher-4764-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3946297 INFO (jetty-launcher-4764-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-27T19:39:33.431Z
[junit4] 2> 3946297 ERROR (jetty-launcher-4764-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3946297 INFO (jetty-launcher-4764-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 3946298 INFO (jetty-launcher-4764-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3946298 INFO (jetty-launcher-4764-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3946298 INFO (jetty-launcher-4764-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-27T19:39:33.432Z
[junit4] 2> 3946301 INFO (zkConnectionManagerCallback-4769-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946301 INFO (zkConnectionManagerCallback-4771-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946302 INFO (jetty-launcher-4764-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3946303 INFO (jetty-launcher-4764-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3946316 INFO (jetty-launcher-4764-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34588/solr
[junit4] 2> 3946316 INFO (jetty-launcher-4764-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34588/solr
[junit4] 2> 3946318 INFO (zkConnectionManagerCallback-4777-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946319 INFO (zkConnectionManagerCallback-4779-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946321 INFO (zkConnectionManagerCallback-4781-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946322 INFO (zkConnectionManagerCallback-4783-thread-1-processing-n:127.0.0.1:54064_solr) [n:127.0.0.1:54064_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946462 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3946463 INFO (jetty-launcher-4764-thread-2) [n:127.0.0.1:54064_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3946463 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:58429_solr
[junit4] 2> 3946464 INFO (jetty-launcher-4764-thread-2) [n:127.0.0.1:54064_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54064_solr
[junit4] 2> 3946465 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.Overseer Overseer (id=99247952579198981-127.0.0.1:58429_solr-n_0000000000) starting
[junit4] 2> 3946468 INFO (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3946470 INFO (zkCallback-4782-thread-1-processing-n:127.0.0.1:54064_solr) [n:127.0.0.1:54064_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3946479 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 3946481 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 3946481 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:54064_solr]
[junit4] 2> 3946482 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 3946482 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 3946482 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58429_solr
[junit4] 2> 3946483 INFO (zkCallback-4782-thread-1-processing-n:127.0.0.1:54064_solr) [n:127.0.0.1:54064_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3946484 INFO (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3946485 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3946485 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion 1
[junit4] 2> 3946486 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 2
[junit4] 2> 3946613 INFO (jetty-launcher-4764-thread-2) [n:127.0.0.1:54064_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54064.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946614 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58429.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946619 INFO (jetty-launcher-4764-thread-2) [n:127.0.0.1:54064_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54064.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946619 INFO (jetty-launcher-4764-thread-2) [n:127.0.0.1:54064_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_54064.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946621 INFO (jetty-launcher-4764-thread-2) [n:127.0.0.1:54064_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.TriggerIntegrationTest_98C4F0A068D35A9F-001/tempDir-001/node2/.
[junit4] 2> 3946622 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58429.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946622 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_58429.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946623 INFO (jetty-launcher-4764-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.TriggerIntegrationTest_98C4F0A068D35A9F-001/tempDir-001/node1/.
[junit4] 2> 3946653 INFO (zkConnectionManagerCallback-4789-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946656 INFO (zkConnectionManagerCallback-4793-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946658 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3946659 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:34588/solr ready
[junit4] 2> 3946784 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeAddedTriggerRestoreState
[junit4] 2> 3946785 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.a.TriggerIntegrationTest /autoscaling.json reset, new znode version 2
[junit4] 2> 3946786 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 3946786 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 3946791 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3946792 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3946792 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 3946797 INFO (qtp1937897085-31620) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3946799 INFO (qtp1298289486-31612) [n:127.0.0.1:58429_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3946800 DEBUG (qtp1937897085-31617) [n:127.0.0.1:54064_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3946800 INFO (qtp1937897085-31617) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 3946801 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 3946801 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3946804 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:58429_solr]
[junit4] 2> 3946804 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerAction}], enabled=true}
[junit4] 2> 3946804 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 3946805 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@659a3066{/solr,null,AVAILABLE}
[junit4] 2> 3946806 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6d25db28{HTTP/1.1,[http/1.1]}{127.0.0.1:60345}
[junit4] 2> 3946806 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.Server Started @3951299ms
[junit4] 2> 3946806 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60345}
[junit4] 2> 3946806 ERROR (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3946806 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 3946806 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3946806 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3946806 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-27T19:39:33.940Z
[junit4] 2> 3946807 INFO (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
[junit4] 2> 3946810 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3946810 INFO (zkConnectionManagerCallback-4795-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946811 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 3946811 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 3946811 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 2
[junit4] 2> 3946811 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3946819 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34588/solr
[junit4] 2> 3946821 INFO (zkConnectionManagerCallback-4799-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946824 INFO (zkConnectionManagerCallback-4801-thread-1-processing-n:127.0.0.1:60345_solr) [n:127.0.0.1:60345_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3946836 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:60345_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3946840 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:60345_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3946843 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:60345_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60345_solr
[junit4] 2> 3946845 INFO (zkCallback-4782-thread-1-processing-n:127.0.0.1:54064_solr) [n:127.0.0.1:54064_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3946845 INFO (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3946847 INFO (zkCallback-4800-thread-1-processing-n:127.0.0.1:60345_solr) [n:127.0.0.1:60345_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3946847 INFO (zkCallback-4792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 3946949 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:60345_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60345.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946956 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:60345_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60345.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946956 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:60345_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_60345.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3946957 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:60345_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.TriggerIntegrationTest_98C4F0A068D35A9F-001/tempDir-001/node3/.
[junit4] 2> 3947821 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 3947822 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3947822 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:60345_solr at time 2147034263632547
[junit4] 2> 3948488 INFO (qtp1937897085-31622) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948489 INFO (qtp1298289486-31613) [n:127.0.0.1:58429_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948492 INFO (qtp1955971146-31672) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948492 DEBUG (qtp1937897085-31621) [n:127.0.0.1:54064_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3948494 INFO (qtp1937897085-31621) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/autoscaling params={wt=javabin&version=2} status=0 QTime=7
[junit4] 2> 3948495 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 4
[junit4] 2> 3948496 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3948496 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$TestTriggerAction}], enabled=true}
[junit4] 2> 3948496 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 4
[junit4] 2> 3948498 INFO (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
[junit4] 2> 3948499 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Adding node from marker path: 127.0.0.1:60345_solr
[junit4] 2> 3948501 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3948501 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_restore_trigger
[junit4] 2> 3948501 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3948501 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_restore_trigger firing registered processor for nodes: [127.0.0.1:60345_solr] added at times [2147034940848485], now=2147034943121521
[junit4] 2> 3948501 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7a0b786f83965Tbs5gkwc63xlxd6sdmd1kwv4d9",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":2147034940848485,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147034940848485],
[junit4] 2> "nodeNames":["127.0.0.1:60345_solr"]}}
[junit4] 2> 3948502 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 4, lastZnodeVersion 4
[junit4] 2> 3948503 INFO (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
[junit4] 2> 3948504 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7a0b786f83965Tbs5gkwc63xlxd6sdmd1kwv4d9",
[junit4] 2> "source":"node_added_restore_trigger",
[junit4] 2> "eventTime":2147034940848485,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147034940848485],
[junit4] 2> "_enqueue_time_":2147034943399442,
[junit4] 2> "nodeNames":["127.0.0.1:60345_solr"]}}
[junit4] 2> 3948505 INFO (TEST-TriggerIntegrationTest.testNodeAddedTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeAddedTriggerRestoreState
[junit4] 2> 3948516 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing took 12 ms for event id=7a0b786f83965Tbs5gkwc63xlxd6sdmd1kwv4d9
[junit4] 2> 3948683 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testTriggerThrottling
[junit4] 2> 3948683 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.a.TriggerIntegrationTest /autoscaling.json reset, new znode version 5
[junit4] 2> 3948684 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 5
[junit4] 2> 3948684 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 5
[junit4] 2> 3948688 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3948688 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3948689 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 5, lastZnodeVersion 5
[junit4] 2> 3948693 INFO (qtp1937897085-31623) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948694 INFO (qtp1298289486-31614) [n:127.0.0.1:58429_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948695 INFO (qtp1955971146-31676) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948695 DEBUG (qtp1955971146-31674) [n:127.0.0.1:60345_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3948696 INFO (qtp1955971146-31674) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=4
[junit4] 2> 3948696 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 6
[junit4] 2> 3948697 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3948697 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 3948697 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 6
[junit4] 2> 3948697 INFO (qtp1937897085-31608) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948698 INFO (qtp1298289486-31616) [n:127.0.0.1:58429_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948698 INFO (qtp1955971146-31670) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3948698 DEBUG (qtp1955971146-31677) [n:127.0.0.1:60345_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3948699 INFO (qtp1955971146-31677) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=3
[junit4] 2> 3948700 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 7
[junit4] 2> 3948700 INFO (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
[junit4] 2> 3948700 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3948700 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 3948700 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3948701 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3948701 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 3948701 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 6
[junit4] 2> 3948701 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 7
[junit4] 2> 3948701 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3948701 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3948704 INFO (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
[junit4] 2> 3948704 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3948705 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3948705 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 7, lastZnodeVersion 7
[junit4] 2> 3948705 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3948705 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3948706 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@55dacf4e{/solr,null,AVAILABLE}
[junit4] 2> 3948707 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@5d54764b{HTTP/1.1,[http/1.1]}{127.0.0.1:33919}
[junit4] 2> 3948707 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.Server Started @3953201ms
[junit4] 2> 3948707 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33919}
[junit4] 2> 3948707 ERROR (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3948707 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 3948707 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3948707 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3948707 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-27T19:39:35.841Z
[junit4] 2> 3948709 INFO (zkConnectionManagerCallback-4805-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3948710 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3948718 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:34588/solr
[junit4] 2> 3948720 INFO (zkConnectionManagerCallback-4809-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3948723 INFO (zkConnectionManagerCallback-4811-thread-1-processing-n:127.0.0.1:33919_solr) [n:127.0.0.1:33919_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3948733 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:33919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 3948736 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:33919_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3948738 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:33919_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33919_solr
[junit4] 2> 3948740 INFO (zkCallback-4782-thread-1-processing-n:127.0.0.1:54064_solr) [n:127.0.0.1:54064_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3948742 INFO (zkCallback-4810-thread-1-processing-n:127.0.0.1:33919_solr) [n:127.0.0.1:33919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3948742 INFO (zkCallback-4800-thread-1-processing-n:127.0.0.1:60345_solr) [n:127.0.0.1:60345_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3948742 INFO (zkCallback-4792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3948743 INFO (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 3948879 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:33919_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33919.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3948885 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:33919_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33919.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3948885 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:33919_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_33919.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@48c04b48
[junit4] 2> 3948886 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [n:127.0.0.1:33919_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.TriggerIntegrationTest_98C4F0A068D35A9F-001/tempDir-001/node4/.
[junit4] 2> 3949703 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3949704 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3949704 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:33919_solr at time 2147036146069577
[junit4] 2> 3949704 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 firing registered processor for nodes: [127.0.0.1:33919_solr] added at times [2147036146069577], now=2147036146172633
[junit4] 2> 3949704 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7a0b7cece7049Tbs5gkwc63xlxd6sdmd1kwv4db",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":2147036146069577,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147036146069577],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3949706 INFO (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
[junit4] 2> 3949706 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7a0b7cece7049Tbs5gkwc63xlxd6sdmd1kwv4db",
[junit4] 2> "source":"node_added_trigger1",
[junit4] 2> "eventTime":2147036146069577,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147036146069577],
[junit4] 2> "_enqueue_time_":2147036146512691,
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3949706 INFO (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest action executed from node_added_trigger1
[junit4] 2> 3949709 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing took 2 ms for event id=7a0b7cece7049Tbs5gkwc63xlxd6sdmd1kwv4db
[junit4] 2> 3950721 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3950721 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3950721 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3950721 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Tracking new node: 127.0.0.1:33919_solr at time 2147037163822023
[junit4] 2> 3950721 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:33919_solr] added at times [2147037163822023], now=2147037163986288
[junit4] 2> 3950721 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3950722 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b80b781bc7Tbs5gkwc63xlxd6sdmd1kwv4dd",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":2147037163822023,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147037163822023],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3950722 INFO (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
[junit4] 2> 3951737 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3951737 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3951737 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3951737 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3951737 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:33919_solr] added at times [2147037163822023], now=2147038179893926
[junit4] 2> 3951737 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b80b781bc7Tbs5gkwc63xlxd6sdmd1kwv4df",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":2147037163822023,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147037163822023],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3952741 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3952741 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3952741 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3952741 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3952741 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:33919_solr] added at times [2147037163822023], now=2147039184524364
[junit4] 2> 3952742 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b80b781bc7Tbs5gkwc63xlxd6sdmd1kwv4dg",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":2147037163822023,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147037163822023],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3953743 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3953743 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3953743 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3953743 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3953743 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:33919_solr] added at times [2147037163822023], now=2147040186742926
[junit4] 2> 3953743 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b80b781bc7Tbs5gkwc63xlxd6sdmd1kwv4dh",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":2147037163822023,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147037163822023],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3954744 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3954744 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3954744 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3954744 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 4
[junit4] 2> 3954744 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 firing registered processor for nodes: [127.0.0.1:33919_solr] added at times [2147037163822023], now=2147041188043373
[junit4] 2> 3954745 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7a0b80b781bc7Tbs5gkwc63xlxd6sdmd1kwv4di",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":2147037163822023,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147037163822023],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3954746 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7a0b80b781bc7Tbs5gkwc63xlxd6sdmd1kwv4di",
[junit4] 2> "source":"node_added_trigger2",
[junit4] 2> "eventTime":2147037163822023,
[junit4] 2> "eventType":"NODEADDED",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147037163822023],
[junit4] 2> "_enqueue_time_":2147041188367959,
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3954747 INFO (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest last action at 1514403576840000000 current time = 1514403581880000000
[junit4] 2> real diff: 5040000000
[junit4] 2> min diff: 4998000000
[junit4] 2> 3954747 INFO (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest action executed from node_added_trigger2
[junit4] 2> 3954750 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing took 3 ms for event id=7a0b80b781bc7Tbs5gkwc63xlxd6sdmd1kwv4di
[junit4] 2> 3954752 INFO (qtp1573688417-31701) [n:127.0.0.1:33919_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954754 INFO (qtp1937897085-31618) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954757 INFO (qtp1298289486-31619) [n:127.0.0.1:58429_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954759 INFO (qtp1955971146-31673) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954760 DEBUG (qtp1955971146-31672) [n:127.0.0.1:60345_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3954761 INFO (qtp1955971146-31672) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=12
[junit4] 2> 3954762 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 8
[junit4] 2> 3954764 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:33919_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3954764 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 3954765 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:33919_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3954766 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:33919_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3954766 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 3954766 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 8
[junit4] 2> 3954766 INFO (qtp1573688417-31703) [n:127.0.0.1:33919_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954768 INFO (qtp1937897085-31617) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954771 INFO (qtp1298289486-31609) [n:127.0.0.1:58429_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954772 INFO (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
[junit4] 2> 3954773 INFO (qtp1955971146-31675) [n:127.0.0.1:60345_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/metrics params={prefix=CORE.coreName&wt=javabin&version=2&group=solr.core} status=0 QTime=0
[junit4] 2> 3954773 DEBUG (qtp1937897085-31620) [n:127.0.0.1:54064_solr ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 3954774 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 8, lastZnodeVersion 8
[junit4] 2> 3954775 INFO (qtp1937897085-31620) [n:127.0.0.1:54064_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/cluster/autoscaling params={wt=javabin&version=2} status=0 QTime=11
[junit4] 2> 3954775 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 4
[junit4] 2> 3954776 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 9
[junit4] 2> 3954777 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:33919_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3954777 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger2 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 3954778 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:33919_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3954779 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:33919_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3954779 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger1 instantiated with properties: {event=nodeAdded, waitFor=0, actions=[{name=test, class=org.apache.solr.cloud.autoscaling.TriggerIntegrationTest$ThrottlingTesterAction}], enabled=true}
[junit4] 2> 3954780 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:54064_solr, 127.0.0.1:33919_solr, 127.0.0.1:58429_solr, 127.0.0.1:60345_solr]
[junit4] 2> 3954780 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 9
[junit4] 2> 3954788 INFO (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest TestTriggerAction init
[junit4] 2> 3954789 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 9, lastZnodeVersion 9
[junit4] 2> 3954791 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 4
[junit4] 2> 3954792 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@5d54764b{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3954792 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=411375952
[junit4] 2> 3954792 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3954792 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@36f36df4: rootName = solr_33919, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@111873a
[junit4] 2> 3954802 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3954802 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7375684: rootName = solr_33919, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@1e8d67f
[junit4] 2> 3954808 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3954808 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@75f2c8d3: rootName = solr_33919, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@5cdb696a
[junit4] 2> 3954811 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:33919_solr
[junit4] 2> 3954812 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3954812 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3954812 INFO (zkCallback-4810-thread-1-processing-n:127.0.0.1:33919_solr) [n:127.0.0.1:33919_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 3954813 INFO (zkCallback-4792-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 3954813 INFO (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 3954814 INFO (zkCallback-4782-thread-1-processing-n:127.0.0.1:54064_solr) [n:127.0.0.1:54064_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 3954814 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160997cae2b000e, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3954814 INFO (zkCallback-4800-thread-1-processing-n:127.0.0.1:60345_solr) [n:127.0.0.1:60345_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (4) -> (3)
[junit4] 2> 3955745 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3955746 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3955748 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3955748 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3955777 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
[junit4] 2> 3955777 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:33919_solr
[junit4] 2> 3955777 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33919_solr]
[junit4] 2> 3955777 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7a0b938e1abc1Tbs5gkwc63xlxd6sdmd1kwv4dj",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2147042220682177,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042220682177],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3955778 INFO (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
[junit4] 2> 3955778 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7a0b938e1abc1Tbs5gkwc63xlxd6sdmd1kwv4dj",
[junit4] 2> "source":"node_lost_trigger1",
[junit4] 2> "eventTime":2147042220682177,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042220682177],
[junit4] 2> "_enqueue_time_":2147042220859416,
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3955778 INFO (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest action executed from node_lost_trigger1
[junit4] 2> 3955779 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing took 1 ms for event id=7a0b938e1abc1Tbs5gkwc63xlxd6sdmd1kwv4dj
[junit4] 2> 3955792 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3
[junit4] 2> 3955792 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Tracking lost node: 127.0.0.1:33919_solr
[junit4] 2> 3955792 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33919_solr]
[junit4] 2> 3955792 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4dl",
[junit4] 2> "source":"node_lost_trigger2",
[junit4] 2> "eventTime":2147042236210260,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042236210260],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3955792 INFO (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.SystemLogListener Collection .system does not exist, disabling logging.
[junit4] 2> 3955792 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:33919_solr] is not ready, will try later
[junit4] 2> 3956314 WARN (zkCallback-4810-thread-1-processing-n:127.0.0.1:33919_solr) [n:127.0.0.1:33919_solr ] o.a.s.c.ZkController Unable to register nodeLost path for 127.0.0.1:33919_solr
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/nodeLost/127.0.0.1:33919_solr
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$create$7(SolrZkClient.java:364)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:364)
[junit4] 2> at org.apache.solr.cloud.ZkController.lambda$registerLiveNodesListener$2(ZkController.java:879)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.lambda$refreshLiveNodes$1(ZkStateReader.java:704)
[junit4] 2> at java.util.concurrent.ConcurrentHashMap$KeySetView.forEach(ConcurrentHashMap.java:4649)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.refreshLiveNodes(ZkStateReader.java:703)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.access$900(ZkStateReader.java:72)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader$LiveNodeWatcher.refreshAndWatch(ZkStateReader.java:1090)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader$LiveNodeWatcher.process(ZkStateReader.java:1085)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$1(SolrZkClient.java:268)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3956315 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@55dacf4e{/solr,null,UNAVAILABLE}
[junit4] 2> 3956752 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3956752 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3956752 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3956752 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3956779 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
[junit4] 2> 3956793 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3
[junit4] 2> 3956793 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33919_solr]
[junit4] 2> 3956794 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4dn",
[junit4] 2> "source":"node_lost_trigger2",
[junit4] 2> "eventTime":2147042236210260,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042236210260],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3956794 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:33919_solr] is not ready, will try later
[junit4] 2> 3957753 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3957753 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3957753 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3957753 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3957780 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
[junit4] 2> 3957794 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3
[junit4] 2> 3957794 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33919_solr]
[junit4] 2> 3957794 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4do",
[junit4] 2> "source":"node_lost_trigger2",
[junit4] 2> "eventTime":2147042236210260,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042236210260],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3957794 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:33919_solr] is not ready, will try later
[junit4] 2> 3958754 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3958754 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3958754 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3958754 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3958780 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
[junit4] 2> 3958794 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3
[junit4] 2> 3958795 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33919_solr]
[junit4] 2> 3958795 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4dp",
[junit4] 2> "source":"node_lost_trigger2",
[junit4] 2> "eventTime":2147042236210260,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042236210260],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3958795 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:33919_solr] is not ready, will try later
[junit4] 2> 3959755 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3959756 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3959756 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3959756 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3959780 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
[junit4] 2> 3959795 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3
[junit4] 2> 3959795 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33919_solr]
[junit4] 2> 3959795 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -------- Cooldown period - rejecting event: {
[junit4] 2> "id":"7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4dq",
[junit4] 2> "source":"node_lost_trigger2",
[junit4] 2> "eventTime":2147042236210260,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042236210260],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3959795 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger processor for lost nodes: [127.0.0.1:33919_solr] is not ready, will try later
[junit4] 2> 3960757 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger2
[junit4] 2> 3960757 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger1
[junit4] 2> 3960757 DEBUG (ScheduledTrigger-11018-thread-1) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3960757 DEBUG (ScheduledTrigger-11018-thread-4) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 3
[junit4] 2> 3960780 DEBUG (ScheduledTrigger-11018-thread-3) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger1 with currently live nodes: 3
[junit4] 2> 3960796 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger2 with currently live nodes: 3
[junit4] 2> 3960796 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.NodeLostTrigger NodeLostTrigger firing registered processor for lost nodes: [127.0.0.1:33919_solr]
[junit4] 2> 3960796 DEBUG (ScheduledTrigger-11018-thread-2) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers ++++++++ Cooldown inactive - processing event: {
[junit4] 2> "id":"7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4dr",
[junit4] 2> "source":"node_lost_trigger2",
[junit4] 2> "eventTime":2147042236210260,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042236210260],
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3960797 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing actions for {
[junit4] 2> "id":"7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4dr",
[junit4] 2> "source":"node_lost_trigger2",
[junit4] 2> "eventTime":2147042236210260,
[junit4] 2> "eventType":"NODELOST",
[junit4] 2> "properties":{
[junit4] 2> "eventTimes":[2147042236210260],
[junit4] 2> "_enqueue_time_":2147047241146965,
[junit4] 2> "nodeNames":["127.0.0.1:33919_solr"]}}
[junit4] 2> 3960798 INFO (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest last action at 1514403582912000000 current time = 1514403587932000000
[junit4] 2> real diff: 5020000000
[junit4] 2> min diff: 4998000000
[junit4] 2> 3960798 INFO (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.TriggerIntegrationTest action executed from node_lost_trigger2
[junit4] 2> 3960798 INFO (TEST-TriggerIntegrationTest.testTriggerThrottling-seed#[98C4F0A068D35A9F]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testTriggerThrottling
[junit4] 2> 3960830 DEBUG (AutoscalingActionExecutor-11019-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.ScheduledTriggers -- processing took 33 ms for event id=7a0b939ce9c54Tbs5gkwc63xlxd6sdmd1kwv4dr
[junit4] 2> 3960961 INFO (TEST-TriggerIntegrationTest.testNodeLostTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testNodeLostTriggerRestoreState
[junit4] 2> 3960961 INFO (TEST-TriggerIntegrationTest.testNodeLostTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.a.TriggerIntegrationTest /autoscaling.json reset, new znode version 10
[junit4] 2> 3960962 DEBUG (zkCallback-4780-thread-1-processing-n:127.0.0.1:58429_solr) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 10
[junit4] 2> 3960962 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 10
[junit4] 2> 3960971 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeLost markers
[junit4] 2> 3960971 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 3960972 DEBUG (OverseerAutoScalingTriggerThread-99247952579198981-127.0.0.1:58429_solr-n_0000000000) [n:127.0.0.1:58429_solr ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 10, lastZnodeVersion 10
[junit4] 2> 3960972 INFO (TEST-TriggerIntegrationTest.testNodeLostTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3960973 INFO (TEST-TriggerIntegrationTest.testNodeLostTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@ced492d{/solr,null,AVAILABLE}
[junit4] 2> 3960973 INFO (TEST-TriggerIntegrationTest.testNodeLostTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@b3ebf76{HTTP/1.1,[http/1.1]}{127.0.0.1:38524}
[junit4] 2> 3960973 INFO (TEST-TriggerIntegrationTest.testNodeLostTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.e.j.s.Server Started @3965471ms
[junit4] 2> 3960973 INFO (TEST-TriggerIntegrationTest.testNodeLostTriggerRestoreState-seed#[98C4F0A068D35A9F]) [ ] o.a.s.c.s.e.JettySolrRunne
[...truncated too long message...]
l.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 4122389 INFO (jetty-closer-4765-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2f5449f1{/solr,null,UNAVAILABLE}
[junit4] 2> 4122393 ERROR (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4122395 INFO (SUITE-TriggerIntegrationTest-seed#[98C4F0A068D35A9F]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34588 34588
[junit4] 2> 4127624 INFO (Thread-8437) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34588 34588
[junit4] 2> 4127641 WARN (Thread-8437) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 20 /solr/aliases.json
[junit4] 2> 20 /solr/clusterprops.json
[junit4] 2> 19 /solr/security.json
[junit4] 2> 3 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 26 /solr/collections/testMetricTrigger/state.json
[junit4] 2> 20 /solr/clusterstate.json
[junit4] 2> 6 /solr/autoscaling.json
[junit4] 2> 3 /solr/overseer_elect/election/99247952579198987-127.0.0.1:60345_solr-n_0000000002
[junit4] 2> 2 /solr/overseer_elect/election/99247952579199005-127.0.0.1:48212_solr-n_0000000008
[junit4] 2> 2 /solr/overseer_elect/election/99247952579199021-127.0.0.1:63784_solr-n_0000000013
[junit4] 2> 2 /solr/overseer_elect/election/99247952579199015-127.0.0.1:65274_solr-n_0000000011
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 20 /solr/live_nodes
[junit4] 2> 20 /solr/collections
[junit4] 2> 6 /solr/overseer/queue-work
[junit4] 2> 6 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 6 /solr/overseer/queue
[junit4] 2> 6 /solr/overseer/collection-queue-work
[junit4] 2> 3 /solr/autoscaling/events/node_added_trigger
[junit4] 2> 2 /solr/autoscaling/events/node_lost_trigger
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.TriggerIntegrationTest_98C4F0A068D35A9F-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {id=Lucene50(blocksize=128), x_s=PostingsFormat(name=MockRandom)}, docValues:{}, maxPointsInLeafNode=1216, maxMBSortInHeap=5.4596507488744495, sim=RandomSimilarity(queryNorm=true): {}, locale=es, timezone=Africa/Timbuktu
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=292399688,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [RuleEngineTest, MultiThreadedOCPTest, ExitableDirectoryReaderTest, QueryResultKeyTest, BadComponentTest, HdfsRecoveryZkTest, TermVectorComponentTest, AnalysisAfterCoreReloadTest, BadCopyFieldTest, TestLMJelinekMercerSimilarityFactory, TestSolrQueryParser, MBeansHandlerTest, DistanceUnitsTest, TestHdfsUpdateLog, TestAuthenticationFramework, InfixSuggestersTest, BufferStoreTest, TestUniqueKeyFieldResource, TestRandomFlRTGCloud, ParsingFieldUpdateProcessorsTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestPKIAuthenticationPlugin, SystemInfoHandlerTest, TestNumericRangeQuery64, TestUtilizeNode, TestMultiWordSynonyms, TestReplicationHandlerBackup, SimpleFacetsTest, BasicZkTest, TestFieldTypeResource, TestCrossCoreJoin, TestCryptoKeys, TestPayloadScoreQParserPlugin, TestRawResponseWriter, TestCharFilters, TestPostingsSolrHighlighter, CollectionsAPIDistributedZkTest, TestPullReplica, TestConfigSetsAPIExclusivity, TestLuceneMatchVersion, ClassificationUpdateProcessorFactoryTest, TestTrie, EchoParamsTest, DocumentAnalysisRequestHandlerTest, TestSolrConfigHandlerConcurrent, DeleteLastCustomShardedReplicaTest, DistributedQueryComponentCustomSortTest, TestCollationField, TestSolrCloudWithKerberosAlt, TestSkipOverseerOperations, TestConfigsApi, LukeRequestHandlerTest, TestInPlaceUpdatesStandalone, DocValuesNotIndexedTest, TestCloudPivotFacet, PolyFieldTest, ZkStateWriterTest, ExternalFileFieldSortTest, TestAnalyzedSuggestions, LeaderInitiatedRecoveryOnCommitTest, TestSimpleTrackingShardHandler, CacheHeaderTest, SuggestComponentContextFilterQueryTest, TestSuggestSpellingConverter, TestMergePolicyConfig, PreAnalyzedFieldTest, TestSimDistributedQueue, TestDistribIDF, ComputePlanActionTest, TestDocSet, TestNodeLostTrigger, TestLegacyFieldCache, CachingDirectoryFactoryTest, DebugComponentTest, WrapperMergePolicyFactoryTest, TestConfig, TestSQLHandlerNonCloud, TestCollapseQParserPlugin, TestNumericTerms64, TestLeaderInitiatedRecoveryThread, TestInitQParser, TestSolrConfigHandler, TestUseDocValuesAsStored, TestRandomFaceting, TestXmlQParser, TestGraphTermsQParserPlugin, JsonLoaderTest, TestQueryTypes, CoreSorterTest, SolrGraphiteReporterTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, HdfsLockFactoryTest, TermsComponentTest, PeerSyncTest, TestGeoJSONResponseWriter, BigEndianAscendingWordSerializerTest, HdfsBasicDistributedZkTest, SolrPluginUtilsTest, ReplaceNodeTest, ChangedSchemaMergeTest, TestCloudInspectUtil, TestStressLucene, SearchHandlerTest, TestGroupingSearch, FieldMutatingUpdateProcessorTest, SSLMigrationTest, SystemLogListenerTest, TestOrdValues, DocValuesMultiTest, OpenCloseCoreStressTest, TestPartialUpdateDeduplication, SparseHLLTest, PKIAuthenticationIntegrationTest, BaseCdcrDistributedZkTest, TestExclusionRuleCollectionAccess, TestOnReconnectListenerSupport, CustomCollectionTest, TestRTGBase, TestSolrJ, TestDistributedStatsComponentCardinality, ZkFailoverTest, VersionInfoTest, SOLR749Test, SpellPossibilityIteratorTest, CollectionsAPIAsyncDistributedZkTest, TestZkAclsWithHadoopAuth, HLLSerializationTest, TestBadConfig, TestSearchPerf, MergeStrategyTest, RecoveryAfterSoftCommitTest, TestRequestForwarding, LoggingHandlerTest, DOMUtilTest, QueryElevationComponentTest, DistributedMLTComponentTest, HdfsChaosMonkeySafeLeaderTest, DirectSolrSpellCheckerTest, TestComplexPhraseQParserPlugin, SpatialRPTFieldTypeTest, TestExtendedDismaxParser, TestLegacyField, ShardSplitTest, OverseerTaskQueueTest, TestElisionMultitermQuery, AutoscalingHistoryHandlerTest, SoftAutoCommitTest, TestFieldSortValues, TestConfigOverlay, ZkSolrClientTest, TestManagedSchemaThreadSafety, CollectionStateFormat2Test, SimplePostToolTest, TestTolerantUpdateProcessorCloud, TestLeaderElectionWithEmptyReplica, MetricTriggerTest, TestHashPartitioner, TestZkChroot, TestSystemIdResolver, TestLuceneIndexBackCompat, AtomicUpdateProcessorFactoryTest, TestJsonFacets, TestMaxScoreQueryParser, QueryEqualityTest, TestFieldCacheVsDocValues, HttpPartitionTest, TestStressReorder, SpellCheckCollatorWithCollapseTest, TestDocBasedVersionConstraints, TestIndexSearcher, TestFieldCache, TestFieldCacheSort, DateRangeFieldTest, TestClusterStateMutator, TestFastLRUCache, UpdateParamsTest, MissingSegmentRecoveryTest, TestCoreBackup, SolrJmxReporterTest, HdfsTlogReplayBufferedWhileIndexingTest, TestPerFieldSimilarity, GraphQueryTest, TestConfigSetProperties, TestSolr4Spatial2, TestDistributedMissingSort, DocumentBuilderTest, InfoHandlerTest, BasicDistributedZkTest, TestReplicationHandler, TestDistributedGrouping, TestFaceting, TermVectorComponentDistributedTest, ZkControllerTest, TestStressVersions, SolrCoreTest, SolrCmdDistributorTest, TestFunctionQuery, TestLazyCores, SignatureUpdateProcessorFactoryTest, CoreAdminHandlerTest, DocValuesTest, TestRemoteStreaming, DistanceFunctionTest, TestSurroundQueryParser, TestQueryUtils, TestReversedWildcardFilterFactory, FileBasedSpellCheckerTest, RequestHandlersTest, TestValueSourceCache, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, UpdateRequestProcessorFactoryTest, QueryParsingTest, CSVRequestHandlerTest, TestComponentsName, HighlighterConfigTest, TestQuerySenderListener, TestSolrIndexConfig, CopyFieldTest, SolrIndexConfigTest, TestSolrDeletionPolicy2, MultiTermTest, SampleTest, TestBinaryField, NumericFieldsTest, OutputWriterTest, DirectSolrConnectionTest, TestPhraseSuggestions, SynonymTokenizerTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestLMDirichletSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, ResourceLoaderTest, TestFastOutputStream, ScriptEngineTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, PrimUtilsTest, DateFieldTest, SpellingQueryConverterTest, TestLRUCache, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, UUIDFieldTest, FileUtilsTest, CircularListTest, CursorPagingTest, SolrTestCaseJ4Test, TestJettySolrRunner, ConnectionReuseTest, AliasIntegrationTest, AssignTest, CloudExitableDirectoryReaderTest, ConcurrentDeleteAndCreateCollectionTest, ConfigSetsAPITest, CreateCollectionCleanupTest, DeleteShardTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, DistributedQueueTest, LeaderFailoverAfterPartitionTest, MoveReplicaHDFSTest, SolrCloudExampleTest, TestCloudRecovery, TestDistribDocBasedVersion, TestLeaderElectionZkExpiry, TestLockTree, TestPullReplicaErrorHandling, TestRequestStatusCollectionAPI, TestSolrCloudWithDelegationTokens, TestSolrCloudWithSecureImpersonation, VMParamsZkACLAndCredentialsProvidersTest, AutoScalingHandlerTest, HdfsAutoAddReplicasIntegrationTest, TriggerIntegrationTest]
[junit4] Completed [579/764 (1!)] on J1 in 181.55s, 14 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 48758 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 355 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/355/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseSerialGC
2 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.AutoAddReplicasPlanActionTest.testSimple
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:50655/solr]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:50655/solr]
at __randomizedtesting.SeedInfo.seed([97917F4C9451E67B:AF225BB2B3A232AA]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.autoscaling.AutoAddReplicasPlanActionTest.testSimple(AutoAddReplicasPlanActionTest.java:110)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50655/solr
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
... 44 more
Caused by: java.net.SocketException: Connection reset
at java.net.SocketInputStream.read(SocketInputStream.java:210)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
... 48 more
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication
Error Message:
Index: 0, Size: 0
Stack Trace:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
at __randomizedtesting.SeedInfo.seed([97917F4C9451E67B:83D92419B7565B65]:0)
at java.util.ArrayList.rangeCheck(ArrayList.java:657)
at java.util.ArrayList.get(ArrayList.java:433)
at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11811 lines...]
[junit4] Suite: org.apache.solr.handler.TestReplicationHandler
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/init-core-data-001
[junit4] 2> 322742 WARN (SUITE-TestReplicationHandler-seed#[97917F4C9451E67B]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=8 numCloses=8
[junit4] 2> 322742 INFO (SUITE-TestReplicationHandler-seed#[97917F4C9451E67B]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 322742 INFO (SUITE-TestReplicationHandler-seed#[97917F4C9451E67B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
[junit4] 2> 322742 INFO (SUITE-TestReplicationHandler-seed#[97917F4C9451E67B]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 322751 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting doTestReplicateAfterStartup
[junit4] 2> 322752 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/collection1
[junit4] 2> 322757 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 322758 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2518fdb0{/solr,null,AVAILABLE}
[junit4] 2> 322761 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@52a19505{HTTP/1.1,[http/1.1]}{127.0.0.1:58971}
[junit4] 2> 322761 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.Server Started @325788ms
[junit4] 2> 322761 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=58971}
[junit4] 2> 322762 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 322762 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 322762 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 322762 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 322762 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-27T07:57:59.184Z
[junit4] 2> 322762 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/solr.xml
[junit4] 2> 322776 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 322776 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 322777 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 322821 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 322834 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 322839 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 322842 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/.
[junit4] 2> 322842 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 322845 INFO (coreLoadExecutor-985-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 322861 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 322881 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 322884 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 322884 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/./collection1, trusted=true
[junit4] 2> 322884 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 322884 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 322884 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/./collection1/data/]
[junit4] 2> 322886 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=37, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 322950 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 322950 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 322951 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=27.4736328125, floorSegmentMB=1.6328125, forceMergeDeletesPctAllowed=23.282325756198887, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6910596725294958
[junit4] 2> 322951 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@107d0d87[collection1] main]
[junit4] 2> 322952 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/collection1/conf
[junit4] 2> 322952 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
[junit4] 2> 322953 INFO (coreLoadExecutor-985-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 322954 INFO (searcherExecutor-986-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@107d0d87[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 323458 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/collection1
[junit4] 2> 323461 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 323463 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4c969422{/solr,null,AVAILABLE}
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@44a07ccd{HTTP/1.1,[http/1.1]}{127.0.0.1:37893}
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.Server Started @326491ms
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=37893}
[junit4] 2> 323464 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-27T07:57:59.886Z
[junit4] 2> 323464 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/solr.xml
[junit4] 2> 323497 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 323497 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 323498 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 323547 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323558 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323558 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323560 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/.
[junit4] 2> 323560 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 323562 INFO (coreLoadExecutor-995-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 323584 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 323591 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 323593 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 323593 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/./collection1, trusted=true
[junit4] 2> 323593 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323594 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 323594 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/./collection1/data/]
[junit4] 2> 323594 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=37, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 323672 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 323672 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 323687 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=27.4736328125, floorSegmentMB=1.6328125, forceMergeDeletesPctAllowed=23.282325756198887, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6910596725294958
[junit4] 2> 323688 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@12be64f9[collection1] main]
[junit4] 2> 323691 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-002/collection1/conf
[junit4] 2> 323706 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
[junit4] 2> 323706 INFO (coreLoadExecutor-995-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 323709 INFO (searcherExecutor-996-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@12be64f9[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 323710 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@44a07ccd{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 323711 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=138734208
[junit4] 2> 323711 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 323711 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2458ac12: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@789129b6
[junit4] 2> 323717 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 323717 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2651be14: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@3a57ebc
[junit4] 2> 323722 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 323723 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7fae3ca4: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@79aea10
[junit4] 2> 323724 INFO (coreCloseExecutor-1001-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@3567a569
[junit4] 2> 323724 INFO (coreCloseExecutor-1001-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=895985001
[junit4] 2> 323725 INFO (coreCloseExecutor-1001-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@25998b72: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@540fe531
[junit4] 2> 323736 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4c969422{/solr,null,UNAVAILABLE}
[junit4] 2> 323743 INFO (qtp1649333230-2662) [ x:collection1] o.a.s.u.DirectUpdateHandler2 [collection1] REMOVING ALL DOCUMENTS FROM INDEX
[junit4] 2> 323746 INFO (qtp1649333230-2662) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{deleteByQuery=*:*} 0 2
[junit4] 2> 323747 INFO (qtp1649333230-2660) [ x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
[junit4] 2> 323747 INFO (qtp1649333230-2660) [ x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@33e4d232 commitCommandVersion:0
[junit4] 2> 323749 INFO (qtp1649333230-2660) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7c1e2005[collection1] main]
[junit4] 2> 323749 INFO (qtp1649333230-2660) [ x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
[junit4] 2> 323752 INFO (searcherExecutor-986-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@7c1e2005[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 323752 INFO (qtp1649333230-2660) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={waitSearcher=true&commit=true&softCommit=false&wt=javabin&version=2}{commit=} 0 4
[junit4] 2> 323755 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@52a19505{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 323755 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1370349047
[junit4] 2> 323756 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 323756 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6248e267: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@3fbd2c7f
[junit4] 2> 323757 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 323757 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@50b8c444: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@3a57ebc
[junit4] 2> 323757 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 323757 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7a5a0a6f: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@79aea10
[junit4] 2> 323760 INFO (coreCloseExecutor-1003-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@18a426ae
[junit4] 2> 323760 INFO (coreCloseExecutor-1003-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=413410990
[junit4] 2> 323760 INFO (coreCloseExecutor-1003-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@399b830c: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@11bbf0a1
[junit4] 2> 323767 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2518fdb0{/solr,null,UNAVAILABLE}
[junit4] 2> 323774 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 323777 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6c7fee61{/solr,null,AVAILABLE}
[junit4] 2> 323780 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4e1a9184{HTTP/1.1,[http/1.1]}{127.0.0.1:55566}
[junit4] 2> 323780 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.e.j.s.Server Started @326807ms
[junit4] 2> 323781 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=55566}
[junit4] 2> 323786 ERROR (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 323786 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 323786 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in standalone mode on port null
[junit4] 2> 323786 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 323786 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-27T07:58:00.208Z
[junit4] 2> 323787 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Loading container configuration from /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/solr.xml
[junit4] 2> 323790 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 323790 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 323791 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 323830 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323839 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323839 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323842 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CorePropertiesLocator Found 1 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/.
[junit4] 2> 323842 INFO (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[97917F4C9451E67B]) [ ] o.a.s.c.CorePropertiesLocator Cores are: [collection1]
[junit4] 2> 323858 INFO (coreLoadExecutor-1010-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 4 transient cores
[junit4] 2> 323887 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 323894 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
[junit4] 2> 323896 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.s.IndexSchema Loaded schema test/1.2 with uniqueid field id
[junit4] 2> 323896 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/./collection1, trusted=true
[junit4] 2> 323896 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.collection1' (registry 'solr.core.collection1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@5fdaeeaf
[junit4] 2> 323896 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 323896 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/collection1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/./collection1/data/]
[junit4] 2> 323898 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=37, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
[junit4] 2> 323949 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 323949 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 323950 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=27.4736328125, floorSegmentMB=1.6328125, forceMergeDeletesPctAllowed=23.282325756198887, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6910596725294958
[junit4] 2> 323951 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@339770ec[collection1] main]
[junit4] 2> 323951 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_97917F4C9451E67B-001/solr-instance-001/collection1/conf
[junit4] 2> 323953 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml
[junit4] 2> 323953 INFO (coreLoadExecutor-1010-thread-1) [ x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 323954 INFO (searcherExecutor-1011-thread-1-processing-x:collection1) [ x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@339770ec[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 323963 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[0]} 0 2
[junit4] 2> 323971 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[1]} 0 5
[junit4] 2> 323973 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[2]} 0 0
[junit4] 2> 323974 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[3]} 0 0
[junit4] 2> 323976 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[4]} 0 0
[junit4] 2> 323977 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[5]} 0 0
[junit4] 2> 323978 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[6]} 0 0
[junit4] 2> 323979 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[7]} 0 0
[junit4] 2> 323980 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[8]} 0 0
[junit4] 2> 323981 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[9]} 0 0
[junit4] 2> 323981 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[10]} 0 0
[junit4] 2> 323982 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[11]} 0 0
[junit4] 2> 323983 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[12]} 0 0
[junit4] 2> 323984 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[13]} 0 0
[junit4] 2> 323984 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[14]} 0 0
[junit4] 2> 323985 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[15]} 0 0
[junit4] 2> 323986 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[16]} 0 0
[junit4] 2> 323986 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[17]} 0 0
[junit4] 2> 323987 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[18]} 0 0
[junit4] 2> 323988 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[19]} 0 0
[junit4] 2> 323988 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[20]} 0 0
[junit4] 2> 323989 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[21]} 0 0
[junit4] 2> 323989 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[22]} 0 0
[junit4] 2> 323990 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[23]} 0 0
[junit4] 2> 323991 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[24]} 0 0
[junit4] 2> 323992 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[25]} 0 0
[junit4] 2> 323992 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[26]} 0 0
[junit4] 2> 323993 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[27]} 0 0
[junit4] 2> 323993 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[28]} 0 0
[junit4] 2> 323994 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[29]} 0 0
[junit4] 2> 323994 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[30]} 0 0
[junit4] 2> 323995 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[31]} 0 0
[junit4] 2> 323996 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[32]} 0 0
[junit4] 2> 323996 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[33]} 0 0
[junit4] 2> 323997 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[34]} 0 0
[junit4] 2> 323998 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[35]} 0 0
[junit4] 2> 323999 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[36]} 0 0
[junit4] 2> 324000 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[37]} 0 0
[junit4] 2> 324000 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[38]} 0 0
[junit4] 2> 324001 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[39]} 0 0
[junit4] 2> 324002 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[40]} 0 0
[junit4] 2> 324003 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[41]} 0 0
[junit4] 2> 324003 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[42]} 0 0
[junit4] 2> 324004 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[43]} 0 0
[junit4] 2> 324005 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[44]} 0 0
[junit4] 2> 324006 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[45]} 0 0
[junit4] 2> 324006 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[46]} 0 0
[junit4] 2> 324007 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[47]} 0 0
[junit4] 2> 324008 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[48]} 0 0
[junit4] 2> 324009 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[49]} 0 0
[junit4] 2> 324010 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[50]} 0 0
[junit4] 2> 324011 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[51]} 0 0
[junit4] 2> 324011 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[52]} 0 0
[junit4] 2> 324012 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[53]} 0 0
[junit4] 2> 324013 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[54]} 0 0
[junit4] 2> 324014 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[55]} 0 0
[junit4] 2> 324015 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[56]} 0 0
[junit4] 2> 324015 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[57]} 0 0
[junit4] 2> 324016 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[58]} 0 0
[junit4] 2> 324016 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[59]} 0 0
[junit4] 2> 324017 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[60]} 0 0
[junit4] 2> 324018 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[61]} 0 0
[junit4] 2> 324018 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[62]} 0 0
[junit4] 2> 324019 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[63]} 0 0
[junit4] 2> 324020 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[64]} 0 0
[junit4] 2> 324021 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[65]} 0 0
[junit4] 2> 324021 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[66]} 0 0
[junit4] 2> 324022 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[67]} 0 0
[junit4] 2> 324023 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[68]} 0 0
[junit4] 2> 324024 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[69]} 0 0
[junit4] 2> 324025 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[70]} 0 0
[junit4] 2> 324025 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[71]} 0 0
[junit4] 2> 324026 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[72]} 0 0
[junit4] 2> 324027 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[73]} 0 0
[junit4] 2> 324027 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[74]} 0 0
[junit4] 2> 324028 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[75]} 0 0
[junit4] 2> 324028 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[76]} 0 0
[junit4] 2> 324029 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[77]} 0 0
[junit4] 2> 324029 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[78]} 0 0
[junit4] 2> 324030 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[79]} 0 0
[junit4] 2> 324031 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[80]} 0 0
[junit4] 2> 324032 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[81]} 0 0
[junit4] 2> 324033 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[82]} 0 0
[junit4] 2> 324033 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[83]} 0 0
[junit4] 2> 324034 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[84]} 0 0
[junit4] 2> 324035 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[85]} 0 0
[junit4] 2> 324035 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[86]} 0 0
[junit4] 2> 324036 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[87]} 0 0
[junit4] 2> 324036 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[88]} 0 0
[junit4] 2> 324037 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[89]} 0 0
[junit4] 2> 324037 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[90]} 0 0
[junit4] 2> 324045 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[91]} 0 0
[junit4] 2> 324046 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[92]} 0 0
[junit4] 2> 324047 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[93]} 0 0
[junit4] 2> 324048 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[94]} 0 0
[junit4] 2> 324048 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[95]} 0 0
[junit4] 2> 324049 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[96]} 0 0
[junit4] 2> 324050 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[97]} 0 0
[junit4] 2> 324050 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[98]} 0 0
[junit4] 2> 324051 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[99]} 0 0
[junit4] 2> 324052 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[100]} 0 0
[junit4] 2> 324052 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[101]} 0 0
[junit4] 2> 324053 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[102]} 0 0
[junit4] 2> 324054 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[103]} 0 0
[junit4] 2> 324054 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[104]} 0 0
[junit4] 2> 324055 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[105]} 0 0
[junit4] 2> 324056 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[106]} 0 0
[junit4] 2> 324057 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[107]} 0 0
[junit4] 2> 324058 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[108]} 0 0
[junit4] 2> 324058 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[109]} 0 0
[junit4] 2> 324060 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[110]} 0 0
[junit4] 2> 324063 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[111]} 0 0
[junit4] 2> 324064 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[112]} 0 0
[junit4] 2> 324064 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[113]} 0 0
[junit4] 2> 324065 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[114]} 0 0
[junit4] 2> 324066 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[115]} 0 0
[junit4] 2> 324067 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[116]} 0 0
[junit4] 2> 324067 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[117]} 0 0
[junit4] 2> 324068 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[118]} 0 0
[junit4] 2> 324069 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[119]} 0 0
[junit4] 2> 324070 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[120]} 0 0
[junit4] 2> 324071 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[121]} 0 0
[junit4] 2> 324072 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[122]} 0 0
[junit4] 2> 324075 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[123]} 0 0
[junit4] 2> 324076 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[124]} 0 0
[junit4] 2> 324077 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[125]} 0 0
[junit4] 2> 324077 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[126]} 0 0
[junit4] 2> 324078 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[127]} 0 0
[junit4] 2> 324079 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[128]} 0 0
[junit4] 2> 324080 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[129]} 0 0
[junit4] 2> 324081 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[130]} 0 0
[junit4] 2> 324082 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[131]} 0 0
[junit4] 2> 324082 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[132]} 0 0
[junit4] 2> 324083 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[133]} 0 0
[junit4] 2> 324083 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[134]} 0 0
[junit4] 2> 324084 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[135]} 0 0
[junit4] 2> 324084 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[136]} 0 0
[junit4] 2> 324085 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[137]} 0 0
[junit4] 2> 324085 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[138]} 0 0
[junit4] 2> 324086 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[139]} 0 0
[junit4] 2> 324086 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[140]} 0 0
[junit4] 2> 324086 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[141]} 0 0
[junit4] 2> 324087 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[142]} 0 0
[junit4] 2> 324088 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[143]} 0 0
[junit4] 2> 324088 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[144]} 0 0
[junit4] 2> 324089 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[145]} 0 0
[junit4] 2> 324089 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[146]} 0 0
[junit4] 2> 324090 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[147]} 0 0
[junit4] 2> 324090 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[148]} 0 0
[junit4] 2> 324091 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[149]} 0 0
[junit4] 2> 324092 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[150]} 0 0
[junit4] 2> 324092 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[151]} 0 0
[junit4] 2> 324093 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[152]} 0 0
[junit4] 2> 324093 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[153]} 0 0
[junit4] 2> 324094 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[154]} 0 0
[junit4] 2> 324095 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[155]} 0 0
[junit4] 2> 324095 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[156]} 0 0
[junit4] 2> 324096 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[157]} 0 0
[junit4] 2> 324096 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[158]} 0 0
[junit4] 2> 324097 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[159]} 0 0
[junit4] 2> 324098 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[160]} 0 0
[junit4] 2> 324098 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[161]} 0 0
[junit4] 2> 324099 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[162]} 0 0
[junit4] 2> 324099 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[163]} 0 0
[junit4] 2> 324100 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[164]} 0 0
[junit4] 2> 324101 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[165]} 0 0
[junit4] 2> 324101 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[166]} 0 0
[junit4] 2> 324102 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[167]} 0 0
[junit4] 2> 324103 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[168]} 0 0
[junit4] 2> 324103 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[169]} 0 0
[junit4] 2> 324104 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[170]} 0 0
[junit4] 2> 324105 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[171]} 0 0
[junit4] 2> 324105 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[172]} 0 0
[junit4] 2> 324106 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[173]} 0 0
[junit4] 2> 324107 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[174]} 0 0
[junit4] 2> 324107 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[175]} 0 0
[junit4] 2> 324108 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[176]} 0 0
[junit4] 2> 324109 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[177]} 0 0
[junit4] 2> 324109 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[178]} 0 0
[junit4] 2> 324110 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[179]} 0 0
[junit4] 2> 324111 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[180]} 0 0
[junit4] 2> 324111 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[181]} 0 0
[junit4] 2> 324112 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[182]} 0 0
[junit4] 2> 324113 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[183]} 0 0
[junit4] 2> 324113 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[184]} 0 0
[junit4] 2> 324114 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[185]} 0 0
[junit4] 2> 324114 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[186]} 0 0
[junit4] 2> 324115 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[187]} 0 0
[junit4] 2> 324115 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[188]} 0 0
[junit4] 2> 324116 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[189]} 0 0
[junit4] 2> 324117 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[190]} 0 0
[junit4] 2> 324117 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[191]} 0 0
[junit4] 2> 324118 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[192]} 0 0
[junit4] 2> 324119 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[193]} 0 0
[junit4] 2> 324120 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[194]} 0 0
[junit4] 2> 324120 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[195]} 0 0
[junit4] 2> 324121 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[196]} 0 0
[junit4] 2> 324122 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[197]} 0 0
[junit4] 2> 324123 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[198]} 0 0
[junit4] 2> 324127 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[199]} 0 0
[junit4] 2> 324128 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[200]} 0 0
[junit4] 2> 324129 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[201]} 0 0
[junit4] 2> 324129 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[202]} 0 0
[junit4] 2> 324130 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[203]} 0 0
[junit4] 2> 324131 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[204]} 0 0
[junit4] 2> 324132 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[205]} 0 0
[junit4] 2> 324132 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[206]} 0 0
[junit4] 2> 324133 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[207]} 0 0
[junit4] 2> 324133 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[208]} 0 0
[junit4] 2> 324134 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[209]} 0 0
[junit4] 2> 324135 INFO (qtp1145960290-2708) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[210]} 0 0
[junit4] 2> 324136 INFO (qtp1145960290-2710) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[211]} 0 0
[junit4] 2> 324136 INFO (qtp1145960290-2707) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[212]} 0 0
[junit4] 2> 324137 INFO (qtp1145960290-2706) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[213]} 0 0
[junit4] 2> 324138 INFO (qtp1145960290-2704) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[214]} 0 0
[junit4] 2> 324138 INFO (qtp1145960290-2709) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[215]} 0 0
[junit4] 2> 324139 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[216]} 0 0
[junit4] 2> 324140 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[217]} 0 0
[junit4] 2> 324140 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[218]} 0 0
[junit4] 2> 324141 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[219]} 0 0
[junit4] 2> 324141 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[220]} 0 0
[junit4] 2> 324141 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[221]} 0 0
[junit4] 2> 324142 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[222]} 0 0
[junit4] 2> 324142 INFO (qtp1145960290-2703) [ x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1] webapp=/solr path=/update params={wt=javabin&version=2}{add=[223]} 0 0
[junit4] 2> 324143 INFO (qt
[...truncated too long message...]
registry solr.core.testSimple2.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@32b048db
[junit4] 2> 1397487 INFO (coreCloseExecutor-4395-thread-5) [n:127.0.0.1:50655_solr c:testSimple3 s:shard3 r:core_node6 x:testSimple3_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testSimple3.shard3.leader, tag=794593712
[junit4] 2> 1397487 INFO (coreCloseExecutor-4395-thread-3) [n:127.0.0.1:50655_solr c:testSimple3 s:shard1 r:core_node2 x:testSimple3_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testSimple3.shard1.replica_n1, tag=87217889
[junit4] 2> 1397487 INFO (coreCloseExecutor-4395-thread-3) [n:127.0.0.1:50655_solr c:testSimple3 s:shard1 r:core_node2 x:testSimple3_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6f2b4c8: rootName = solr_50655, domain = solr.core.testSimple3.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.testSimple3.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@f92a20e
[junit4] 2> 1397499 INFO (coreCloseExecutor-4396-thread-3) [n:127.0.0.1:44783_solr c:testSimple2 s:shard1 r:core_node4 x:testSimple2_shard1_replica_n2] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testSimple2.shard1.leader, tag=2119066137
[junit4] 2> 1397502 INFO (jetty-closer-1935-thread-2) [ ] o.a.s.c.Overseer Overseer (id=99245263411216392-127.0.0.1:44783_solr-n_0000000000) closing
[junit4] 2> 1397502 INFO (OverseerStateUpdate-99245263411216392-127.0.0.1:44783_solr-n_0000000000) [n:127.0.0.1:44783_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:44783_solr
[junit4] 2> 1397504 INFO (coreCloseExecutor-4395-thread-3) [n:127.0.0.1:50655_solr c:testSimple3 s:shard1 r:core_node2 x:testSimple3_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testSimple3.shard1.leader, tag=87217889
[junit4] 2> 1397504 INFO (coreCloseExecutor-4395-thread-4) [n:127.0.0.1:50655_solr c:testSimple3 s:shard2 r:core_node5 x:testSimple3_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testSimple3.shard2.replica_n3, tag=1095218848
[junit4] 2> 1397504 INFO (coreCloseExecutor-4395-thread-4) [n:127.0.0.1:50655_solr c:testSimple3 s:shard2 r:core_node5 x:testSimple3_shard2_replica_n3] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7fed0df: rootName = solr_50655, domain = solr.core.testSimple3.shard2.replica_n3, service url = null, agent id = null] for registry solr.core.testSimple3.shard2.replica_n3 / com.codahale.metrics.MetricRegistry@1fb28beb
[junit4] 2> 1397506 WARN (OverseerAutoScalingTriggerThread-99245263411216392-127.0.0.1:44783_solr-n_0000000000) [n:127.0.0.1:44783_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 1397515 INFO (zkCallback-1980-thread-3-processing-n:127.0.0.1:50655_solr) [n:127.0.0.1:50655_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:50655_solr
[junit4] 2> 1397517 INFO (jetty-closer-1935-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@79530a4f{/solr,null,UNAVAILABLE}
[junit4] 2> 1397520 INFO (coreCloseExecutor-4395-thread-4) [n:127.0.0.1:50655_solr c:testSimple3 s:shard2 r:core_node5 x:testSimple3_shard2_replica_n3] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testSimple3.shard2.leader, tag=1095218848
[junit4] 2> 1397525 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160970a8f44000e, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 1397526 INFO (jetty-closer-1935-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@76c177f4{/solr,null,UNAVAILABLE}
[junit4] 2> 1397528 ERROR (SUITE-AutoAddReplicasPlanActionTest-seed#[97917F4C9451E67B]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 1397528 INFO (SUITE-AutoAddReplicasPlanActionTest-seed#[97917F4C9451E67B]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37251 37251
[junit4] 2> 1407679 INFO (Thread-2046) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:37251 37251
[junit4] 2> 1407680 WARN (Thread-2046) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 32 /solr/collections/testSimple1/state.json
[junit4] 2> 14 /solr/collections/testSimple3/state.json
[junit4] 2> 13 /solr/collections/testSimple2/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 3 /solr/overseer_elect/election/99245263411216392-127.0.0.1:44783_solr-n_0000000000
[junit4] 2> 2 /solr/collections/testSimple1/leader_elect/shard2/election/99245263411216392-core_node7-n_0000000000
[junit4] 2> 2 /solr/collections/testSimple1/leader_elect/shard1/election/99245263411216392-core_node3-n_0000000000
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.AutoAddReplicasPlanActionTest_97917F4C9451E67B-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=860, maxMBSortInHeap=7.332970119887788, sim=RandomSimilarity(queryNorm=true): {}, locale=fi-FI, timezone=CNT
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=39899776,total=385118208
[junit4] 2> NOTE: All tests run in this JVM: [DistributedQueueTest, AtomicUpdateProcessorFactoryTest, TestRequestForwarding, ZkSolrClientTest, TestPostingsSolrHighlighter, TestMaxTokenLenTokenizer, TestPKIAuthenticationPlugin, InfoHandlerTest, TestReload, FileUtilsTest, TestComponentsName, SimpleFacetsTest, CollectionsAPIAsyncDistributedZkTest, TestElisionMultitermQuery, TestOverriddenPrefixQueryForCustomFieldType, TestRawResponseWriter, TestXmlQParser, CloudMLTQParserTest, DeleteReplicaTest, TestRestManager, SolrGraphiteReporterTest, TestExactStatsCache, SSLMigrationTest, SolrCloudReportersTest, TestRestoreCore, TestNodeLostTrigger, TestMergePolicyConfig, BlockCacheTest, HdfsSyncSliceTest, TestUtilizeNode, TestSimDistributedQueue, TestSchemaResource, TestCloudRecovery, TestSort, TestUniqueKeyFieldResource, ZkStateWriterTest, DirectSolrSpellCheckerTest, TestConfigSetProperties, TriLevelCompositeIdRoutingTest, TestCSVLoader, IndexSchemaRuntimeFieldTest, TestConfigSetsAPIExclusivity, PolyFieldTest, SolrCloudExampleTest, DeleteStatusTest, TestNumericTerms64, TestClassicSimilarityFactory, TestSolrJ, TestDocBasedVersionConstraints, TestRandomFaceting, TestDistribDocBasedVersion, ConcurrentDeleteAndCreateCollectionTest, LeaderFailoverAfterPartitionTest, TestSuggestSpellingConverter, TestClusterStateMutator, TestMaxScoreQueryParser, TestCloudManagedSchema, TestMultiWordSynonyms, MoreLikeThisHandlerTest, DeleteLastCustomShardedReplicaTest, AddBlockUpdateTest, SuggestComponentContextFilterQueryTest, SystemInfoHandlerTest, SystemLogListenerTest, OverseerTaskQueueTest, TestInfoStreamLogging, ChaosMonkeyNothingIsSafeWithPullReplicasTest, TestLargeCluster, HdfsTlogReplayBufferedWhileIndexingTest, TestInPlaceUpdatesStandalone, TestFastWriter, PingRequestHandlerTest, TestBlobHandler, TestQueryTypes, CollectionStateFormat2Test, TestExecutePlanAction, TermsComponentTest, HdfsWriteToMultipleCollectionsTest, ExplicitHLLTest, DebugComponentTest, RuleEngineTest, CoreAdminHandlerTest, TestDynamicFieldResource, CoreSorterTest, TestLeaderElectionWithEmptyReplica, CdcrVersionReplicationTest, HdfsAutoAddReplicasIntegrationTest, TestFoldingMultitermQuery, TestPullReplica, TestAnalyzedSuggestions, SolrMetricsIntegrationTest, MetricsConfigTest, TestRTimerTree, DocExpirationUpdateProcessorFactoryTest, InfixSuggestersTest, TestIndexSearcher, TestNRTOpen, CoreAdminRequestStatusTest, TolerantUpdateProcessorTest, TestDistributedMissingSort, MetricTriggerTest, PKIAuthenticationIntegrationTest, TestStreamBody, DOMUtilTest, TestCharFilters, DocValuesMultiTest, CursorMarkTest, GraphQueryTest, TestPerFieldSimilarity, TestCoreBackup, QueryResultKeyTest, DistributedFacetSimpleRefinementLongTailTest, PropertiesRequestHandlerTest, DistributedMLTComponentTest, DocumentBuilderTest, ChangedSchemaMergeTest, TestRandomFlRTGCloud, TestBlendedInfixSuggestions, SolrJmxReporterTest, TestComputePlanAction, SampleTest, SpellingQueryConverterTest, TestCustomDocTransformer, TestDistributedGrouping, TestManagedSchema, TestSha256AuthenticationProvider, VersionInfoTest, TestBM25SimilarityFactory, BasicZkTest, TestTrie, LeaderInitiatedRecoveryOnCommitTest, TestSubQueryTransformerCrossCore, AutoAddReplicasPlanActionTest]
[junit4] Completed [238/764 (2!)] on J1 in 24.37s, 1 test, 1 error <<< FAILURES!
[...truncated 49953 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 354 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/354/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode
Error Message:
no IGNORED events
Stack Trace:
java.lang.AssertionError: no IGNORED events
at __randomizedtesting.SeedInfo.seed([EEDF50B4784E0910:49304D17B7038608]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.apache.solr.cloud.autoscaling.sim.TestLargeCluster.testAddNode(TestLargeCluster.java:267)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 11625 lines...]
[junit4] Suite: org.apache.solr.cloud.autoscaling.sim.TestLargeCluster
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_EEDF50B4784E0910-001/init-core-data-001
[junit4] 2> 0 INFO (SUITE-TestLargeCluster-seed#[EEDF50B4784E0910]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 70 INFO (SUITE-TestLargeCluster-seed#[EEDF50B4784E0910]-worker) [ ] o.e.j.u.log Logging initialized @1852ms
[junit4] 2> 76 INFO (SUITE-TestLargeCluster-seed#[EEDF50B4784E0910]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 99 INFO (SUITE-TestLargeCluster-seed#[EEDF50B4784E0910]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 259 INFO (SUITE-TestLargeCluster-seed#[EEDF50B4784E0910]-worker) [ ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
[junit4] 2> 425 INFO (TEST-TestLargeCluster.testAddNode-seed#[EEDF50B4784E0910]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testAddNode
[junit4] 2> 440 INFO (TEST-TestLargeCluster.testAddNode-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimCloudManager === Restarting OverseerTriggerThread and clearing object cache...
[junit4] 2> 441 DEBUG (TEST-TestLargeCluster.testAddNode-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> 454 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 460 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 1
[junit4] 2> 523 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Adding .autoAddReplicas trigger
[junit4] 2> 550 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 551 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 1, lastZnodeVersion -1
[junit4] 2> 551 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 1
[junit4] 2> 552 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 2
[junit4] 2> 553 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 561 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 561 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 1
[junit4] 2> 561 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 2
[junit4] 2> 570 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread -- clean old nodeAdded markers
[junit4] 2> 570 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 2, lastZnodeVersion 2
[junit4] 2> 579 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 626 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 658 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 678 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 700 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 720 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 741 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 771 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 777 DEBUG (simCloudManagerPool-5-thread-1) [ ] o.a.s.c.a.AutoScalingHandler Verified autoscaling configuration
[junit4] 2> 779 DEBUG (simCloudManagerPool-5-thread-1) [ ] o.a.s.c.a.OverseerTriggerThread Refreshing /autoscaling.json with znode version 3
[junit4] 2> 780 DEBUG (simCloudManagerPool-5-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 784 DEBUG (simCloudManagerPool-5-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Initial livenodes: [127.0.0.1:10082_solr, 127.0.0.1:10004_solr, 127.0.0.1:10018_solr, 127.0.0.1:10049_solr, 127.0.0.1:10020_solr, 127.0.0.1:10065_solr, 127.0.0.1:10021_solr, 127.0.0.1:10005_solr, 127.0.0.1:10066_solr, 127.0.0.1:10035_solr, 127.0.0.1:10051_solr, 127.0.0.1:10019_solr, 127.0.0.1:10096_solr, 127.0.0.1:10022_solr, 127.0.0.1:10033_solr, 127.0.0.1:10017_solr, 127.0.0.1:10047_solr, 127.0.0.1:10095_solr, 127.0.0.1:10081_solr, 127.0.0.1:10084_solr, 127.0.0.1:10036_solr, 127.0.0.1:10052_solr, 127.0.0.1:10006_solr, 127.0.0.1:10079_solr, 127.0.0.1:10098_solr, 127.0.0.1:10003_solr, 127.0.0.1:10001_solr, 127.0.0.1:10054_solr, 127.0.0.1:10099_solr, 127.0.0.1:10071_solr, 127.0.0.1:10023_solr, 127.0.0.1:10076_solr, 127.0.0.1:10007_solr, 127.0.0.1:10055_solr, 127.0.0.1:10010_solr, 127.0.0.1:10016_solr, 127.0.0.1:10070_solr, 127.0.0.1:10077_solr, 127.0.0.1:10038_solr, 127.0.0.1:10032_solr, 127.0.0.1:10093_solr, 127.0.0.1:10061_solr, 127.0.0.1:10067_solr, 127.0.0.1:10086_solr, 127.0.0.1:10089_solr, 127.0.0.1:10045_solr, 127.0.0.1:10042_solr, 127.0.0.1:10029_solr, 127.0.0.1:10083_solr, 127.0.0.1:10080_solr, 127.0.0.1:10048_solr, 127.0.0.1:10064_solr, 127.0.0.1:10043_solr, 127.0.0.1:10057_solr, 127.0.0.1:10074_solr, 127.0.0.1:10026_solr, 127.0.0.1:10012_solr, 127.0.0.1:10060_solr, 127.0.0.1:10073_solr, 127.0.0.1:10058_solr, 127.0.0.1:10013_solr, 127.0.0.1:10090_solr, 127.0.0.1:10088_solr, 127.0.0.1:10027_solr, 127.0.0.1:10039_solr, 127.0.0.1:10025_solr, 127.0.0.1:10044_solr, 127.0.0.1:10028_solr, 127.0.0.1:10092_solr, 127.0.0.1:10041_solr, 127.0.0.1:10087_solr, 127.0.0.1:10009_solr, 127.0.0.1:10000_solr, 127.0.0.1:10030_solr, 127.0.0.1:10014_solr, 127.0.0.1:10011_solr, 127.0.0.1:10040_solr, 127.0.0.1:10085_solr, 127.0.0.1:10046_solr, 127.0.0.1:10068_solr, 127.0.0.1:10063_solr, 127.0.0.1:10015_solr, 127.0.0.1:10069_solr, 127.0.0.1:10062_solr, 127.0.0.1:10008_solr, 127.0.0.1:10024_solr, 127.0.0.1:10050_solr, 127.0.0.1:10075_solr, 127.0.0.1:10078_solr, 127.0.0.1:10002_solr, 127.0.0.1:10097_solr, 127.0.0.1:10031_solr, 127.0.0.1:10034_solr, 127.0.0.1:10091_solr, 127.0.0.1:10094_solr, 127.0.0.1:10037_solr, 127.0.0.1:10053_solr, 127.0.0.1:10059_solr, 127.0.0.1:10056_solr, 127.0.0.1:10072_solr]
[junit4] 2> 785 DEBUG (simCloudManagerPool-5-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger NodeAddedTrigger node_added_trigger instantiated with properties: {event=nodeAdded, waitFor=5, actions=[{name=compute, class=org.apache.solr.cloud.autoscaling.ComputePlanAction}, {name=execute, class=org.apache.solr.cloud.autoscaling.ExecutePlanAction}], enabled=true}
[junit4] 2> 786 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Processed trigger updates upto znodeVersion 3
[junit4] 2> 787 DEBUG (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread Current znodeVersion 3, lastZnodeVersion 3
[junit4] 2> 788 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 788 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 792 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 812 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 813 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 813 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 843 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 844 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 844 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 869 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 869 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 869 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 890 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 891 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 892 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 910 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 914 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 914 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 978 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 978 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 978 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 998 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 998 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 998 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1018 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1019 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1019 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1039 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1039 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1039 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1059 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1059 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1059 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1080 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1080 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1083 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1103 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1103 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1103 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1124 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1125 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1124 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1147 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1147 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1148 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1168 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1168 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1170 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1190 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1190 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1190 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1210 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1210 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1211 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1231 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1231 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1231 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1251 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1251 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1251 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1272 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1272 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1272 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1292 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1292 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1292 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1312 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1313 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1313 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1333 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1333 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1333 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1354 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1354 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1354 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1374 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1374 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1374 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1395 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1395 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1395 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1415 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1415 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1416 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1436 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1436 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1436 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1458 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1459 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1458 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1479 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1480 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1480 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1500 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1502 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1505 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1521 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1525 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1525 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1543 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1546 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1546 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1563 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1566 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1566 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1583 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1586 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1586 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1604 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1607 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1607 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1624 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1627 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1627 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1645 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1648 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1648 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1665 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1668 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1668 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1686 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1688 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1688 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1706 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1709 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1709 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1727 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1729 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1729 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1747 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1750 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1750 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1768 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1771 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1771 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1788 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1791 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1791 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1808 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1812 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1812 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1829 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1832 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1832 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1849 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1852 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1852 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1870 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1873 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1873 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1890 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1893 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1893 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1910 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1913 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1914 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1931 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1934 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1934 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1951 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1954 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1954 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1972 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1977 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1977 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 1992 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 1998 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 1998 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2013 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2018 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2018 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2033 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2039 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2039 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2054 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2059 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2059 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2074 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2080 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2080 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2095 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2100 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2100 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2115 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2121 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2121 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2137 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2142 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2142 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2158 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2162 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2162 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2180 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2183 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2183 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2201 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2203 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2203 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2221 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2223 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2224 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2241 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2244 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2244 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2262 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2265 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2266 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2284 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2286 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2286 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2304 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2306 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2307 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2324 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2327 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2327 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2345 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2353 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2353 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2365 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2373 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2373 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2386 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2393 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2394 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2406 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2414 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2414 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2426 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2435 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2435 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2447 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2455 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2455 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2467 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2475 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2475 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2487 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2496 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2496 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2508 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2516 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2516 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2528 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2537 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2537 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2549 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2557 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2557 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2569 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2577 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2578 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2589 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2598 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2598 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2610 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2618 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2618 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2630 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2639 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2639 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2650 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2659 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2659 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2671 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2679 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2680 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2691 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2700 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2700 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2711 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2720 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2720 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2732 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2741 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2741 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2753 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2761 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2761 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2774 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2782 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2782 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2794 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2802 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2802 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2815 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2822 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2822 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2835 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2843 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2843 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2856 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2863 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2863 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2877 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2883 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2884 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2897 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2904 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2904 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2920 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard2: {"core_node39":{
[junit4] 2> "node_name":"127.0.0.1:10070_solr",
[junit4] 2> "core":"testNodeAdded_shard2_replica_n39",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2920 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard4: {"core_node98":{
[junit4] 2> "node_name":"127.0.0.1:10034_solr",
[junit4] 2> "core":"testNodeAdded_shard4_replica_n98",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard4",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2921 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard5: {"core_node122":{
[junit4] 2> "node_name":"127.0.0.1:10086_solr",
[junit4] 2> "core":"testNodeAdded_shard5_replica_n122",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard5",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2921 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard1: {"core_node18":{
[junit4] 2> "node_name":"127.0.0.1:10090_solr",
[junit4] 2> "core":"testNodeAdded_shard1_replica_t18",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2921 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard6: {"core_node154":{
[junit4] 2> "node_name":"127.0.0.1:10020_solr",
[junit4] 2> "core":"testNodeAdded_shard6_replica_n154",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard6",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2921 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard7: {"core_node200":{
[junit4] 2> "node_name":"127.0.0.1:10083_solr",
[junit4] 2> "core":"testNodeAdded_shard7_replica_t200",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard7",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2921 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard10: {"core_node278":{
[junit4] 2> "node_name":"127.0.0.1:10068_solr",
[junit4] 2> "core":"testNodeAdded_shard10_replica_n278",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"NRT",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard10",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2921 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard8: {"core_node230":{
[junit4] 2> "node_name":"127.0.0.1:10012_solr",
[junit4] 2> "core":"testNodeAdded_shard8_replica_t230",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard8",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2922 DEBUG (simCloudManagerPool-5-thread-50) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard9: {"core_node254":{
[junit4] 2> "node_name":"127.0.0.1:10020_solr",
[junit4] 2> "core":"testNodeAdded_shard9_replica_t254",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard9",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2925 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2925 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2925 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2925 DEBUG (simCloudManagerPool-5-thread-2) [ ] o.a.s.c.a.s.SimClusterStateProvider -- elected new leader for testNodeAdded / shard3: {"core_node76":{
[junit4] 2> "node_name":"127.0.0.1:10019_solr",
[junit4] 2> "core":"testNodeAdded_shard3_replica_t76",
[junit4] 2> "leader":"true",
[junit4] 2> "state":"active",
[junit4] 2> "type":"TLOG",
[junit4] 2> "INDEX.sizeInBytes":123450000,
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"testNodeAdded"}}
[junit4] 2> 2945 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2945 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2945 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2965 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2965 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2966 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 2986 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 2986 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 2986 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3007 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3007 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3007 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3027 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3027 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3027 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3047 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3048 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3048 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3068 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3068 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3068 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3088 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3088 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3088 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3109 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3109 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3109 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3129 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3129 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3129 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3149 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3149 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3149 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3170 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3170 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3170 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3190 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3190 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3190 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3211 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3211 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3211 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3231 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3231 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3231 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3251 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3251 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3251 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3272 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3272 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3272 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3292 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3292 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3292 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3312 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3312 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3312 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3332 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3333 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3333 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3353 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3353 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3353 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3373 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3373 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3373 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3394 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3394 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3394 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3414 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3414 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3414 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3434 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3435 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3435 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3455 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3455 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3455 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3477 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3477 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3477 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3498 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3498 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3498 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3518 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3519 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3519 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3539 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3539 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3539 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3559 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3559 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3559 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3580 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3580 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3580 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3600 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3600 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3601 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3621 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3621 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3621 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3641 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3641 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3641 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3662 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3662 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3662 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3682 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3682 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3682 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3702 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3702 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3703 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3723 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3723 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3723 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3743 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3743 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3743 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3764 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3764 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3764 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3784 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3784 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3784 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3804 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3804 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3804 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3825 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3825 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3825 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3845 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3846 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3846 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3867 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3867 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3867 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3887 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3888 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3888 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3908 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3908 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3908 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3928 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3928 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3928 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3948 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3948 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3948 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3969 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3969 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3969 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 3989 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 3989 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 3989 DEBUG (ScheduledTrigger-6-thread-3) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 4010 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 4010 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 4010 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 4030 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 4030 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 4030 DEBUG (ScheduledTrigger-6-thread-2) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 4050 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 4051 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 4050 DEBUG (ScheduledTrigger-6-thread-1) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 100
[junit4] 2> 4071 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Running NodeAddedTrigger node_added_trigger
[junit4] 2> 4071 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeAddedTrigger Found livenodes: 100
[junit4] 2> 4071 DEBUG (ScheduledTrigger-6-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLos
[...truncated too long message...]
er with currently live nodes: 80
[junit4] 2> 27043 DEBUG (ScheduledTrigger-12-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 80
[junit4] 2> 27044 DEBUG (ScheduledTrigger-12-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 80
[junit4] 2> 27067 DEBUG (ScheduledTrigger-12-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: node_lost_trigger with currently live nodes: 80
[junit4] 2> 27067 DEBUG (ScheduledTrigger-12-thread-4) [ ] o.a.s.c.a.NodeLostTrigger Running NodeLostTrigger: .auto_add_replicas with currently live nodes: 80
[junit4] 2> 27070 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.TestLargeCluster #### System stabilized after 71587 ms
[junit4] 2> 27070 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testNodeLost
[junit4] 2> 27070 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase
[junit4] 2>
[junit4] 2> 27070 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2> 27070 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ############ FINAL CLUSTER STATS ############
[junit4] 2> 27070 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase #############################################
[junit4] 2>
[junit4] 2> 27070 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Live nodes: 80
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Empty nodes: 0
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Dead nodes: 20
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10004_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10005_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10020_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10021_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10033_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10035_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10036_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10047_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10049_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10051_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10065_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10082_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10095_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10096_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10102_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10104_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10105_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10119_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10120_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## 127.0.0.1:10121_solr
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Collections: [testNodeLost]
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Max replicas per node: 3
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Min replicas per node: 2
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## Total replicas: 200
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * testNodeLost 200
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - active 200
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Final Solr op counts ##########
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - CREATE 1
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - DELETESTATUS 40
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - MOVEREPLICA 40
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - REQUESTSTATUS 40
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - autoscaling 1
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - update 60
[junit4] 2> 27071 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ######### Autoscaling event counts ###########
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: .auto_add_replicas
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - IGNORED 19
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 4
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 4
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## * Trigger: node_lost_trigger
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - IGNORED 25
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - STARTED 4
[junit4] 2> 27072 INFO (TEST-TestLargeCluster.testNodeLost-seed#[EEDF50B4784E0910]) [ ] o.a.s.c.a.s.SimSolrCloudTestCase ## - SUCCEEDED 4
[junit4] 2> 27074 WARN (Simulated OverseerAutoScalingTriggerThread) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 27079 DEBUG (SUITE-TestLargeCluster-seed#[EEDF50B4784E0910]-worker) [ ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed explicitly
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.autoscaling.sim.TestLargeCluster_EEDF50B4784E0910-001
[junit4] 2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=4, maxDocsPerChunk=828, blockSize=8), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=4, blockSize=8)), sim=RandomSimilarity(queryNorm=false): {}, locale=es-SV, timezone=Europe/Zurich
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=98590400,total=236978176
[junit4] 2> NOTE: All tests run in this JVM: [RegexBytesRefFilterTest, TestLargeCluster]
[junit4] Completed [3/764 (1!)] on J1 in 27.57s, 4 tests, 1 failure, 1 skipped <<< FAILURES!
[...truncated 50699 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 353 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/353/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation: 1) Thread[id=30907, name=jetty-launcher-6901-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 2) Thread[id=30916, name=jetty-launcher-6901-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
1) Thread[id=30907, name=jetty-launcher-6901-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
2) Thread[id=30916, name=jetty-launcher-6901-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
at __randomizedtesting.SeedInfo.seed([FF4EC3AE66AAE64A]:0)
Build Log:
[...truncated 13473 lines...]
[junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation
[junit4] 2> 3094979 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_FF4EC3AE66AAE64A-001/init-core-data-001
[junit4] 2> 3094979 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 3094981 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 3095022 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_FF4EC3AE66AAE64A-001/tempDir-001
[junit4] 2> 3095022 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3095022 INFO (Thread-7989) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3095023 INFO (Thread-7989) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3095024 ERROR (Thread-7989) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3095126 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:49813
[junit4] 2> 3095130 INFO (zkConnectionManagerCallback-6904-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095138 INFO (jetty-launcher-6901-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3095140 INFO (jetty-launcher-6901-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 3095141 INFO (jetty-launcher-6901-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5af889ba{/solr,null,AVAILABLE}
[junit4] 2> 3095143 INFO (jetty-launcher-6901-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@78519b61{HTTP/1.1,[http/1.1]}{127.0.0.1:40979}
[junit4] 2> 3095143 INFO (jetty-launcher-6901-thread-1) [ ] o.e.j.s.Server Started @3098921ms
[junit4] 2> 3095143 INFO (jetty-launcher-6901-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40979}
[junit4] 2> 3095143 INFO (jetty-launcher-6901-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@30bbc5a9{/solr,null,AVAILABLE}
[junit4] 2> 3095143 ERROR (jetty-launcher-6901-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3095143 INFO (jetty-launcher-6901-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@79f7cdfc{HTTP/1.1,[http/1.1]}{127.0.0.1:62976}
[junit4] 2> 3095143 INFO (jetty-launcher-6901-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-2) [ ] o.e.j.s.Server Started @3098921ms
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=62976}
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-26T10:58:51.034Z
[junit4] 2> 3095144 ERROR (jetty-launcher-6901-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr™ version 7.3.0
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 3095144 INFO (jetty-launcher-6901-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-26T10:58:51.034Z
[junit4] 2> 3095147 INFO (zkConnectionManagerCallback-6906-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095147 INFO (zkConnectionManagerCallback-6908-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095148 INFO (jetty-launcher-6901-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3095149 INFO (jetty-launcher-6901-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 3095158 INFO (jetty-launcher-6901-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49813/solr
[junit4] 2> 3095158 INFO (jetty-launcher-6901-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:49813/solr
[junit4] 2> 3095160 INFO (zkConnectionManagerCallback-6913-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095160 INFO (zkConnectionManagerCallback-6916-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095163 INFO (zkConnectionManagerCallback-6920-thread-1-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095164 INFO (zkConnectionManagerCallback-6918-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095408 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3095410 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:62976_solr
[junit4] 2> 3095414 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3095416 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40979_solr
[junit4] 2> 3095418 INFO (zkCallback-6917-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3095424 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.Overseer Overseer (id=99240242768379910-127.0.0.1:62976_solr-n_0000000000) starting
[junit4] 2> 3095424 INFO (zkCallback-6919-thread-1-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3095441 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62976_solr
[junit4] 2> 3095443 INFO (zkCallback-6919-thread-1-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3095444 INFO (zkCallback-6917-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 3095494 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 3095494 WARN (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3095495 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 3095496 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 3095499 INFO (jetty-launcher-6901-thread-1-EventThread) [n:127.0.0.1:40979_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 3095525 WARN (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3095526 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HttpParamDelegationTokenPlugin
[junit4] 2> 3095526 WARN (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3095526 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.s.KerberosPlugin Params: {solr.kerberos.impersonator.user.noGroups.hosts=*, solr.kerberos.impersonator.user.bogusGroup.groups=__some_bogus_group, token.valid=30, cookie.domain=127.0.0.1, solr.kerberos.impersonator.user.anyHostAnyUser.groups=*, solr.kerberos.impersonator.user.bogusGroup.hosts=*, solr.kerberos.impersonator.user.anyHostAnyUser.hosts=*, token.validity=36000, solr.kerberos.impersonator.user.noHosts.groups=*, type=kerberos, solr.kerberos.impersonator.user.localHostAnyGroup.groups=*, delegation-token.token-kind=solr-dt, solr.kerberos.impersonator.user.wrongHost.hosts=1.1.1.1.1.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, solr.kerberos.impersonator.user.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, signer.secret.provider.zookeeper.path=/token, solr.kerberos.impersonator.user.anyHostUsersGroup.groups=staff, zk-dt-secret-manager.enable=true, solr.kerberos.impersonator.user.wrongHost.groups=*, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper, solr.kerberos.impersonator.user.anyHostUsersGroup.hosts=*}
[junit4] 2> 3095528 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 3095536 INFO (jetty-launcher-6901-thread-2-EventThread) [n:127.0.0.1:62976_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 3095560 WARN (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.s.i.Krb5HttpClientBuilder org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder is configured without specifying system property 'java.security.auth.login.config'
[junit4] 2> 3095592 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40979.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e6c76b2
[junit4] 2> 3095602 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40979.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e6c76b2
[junit4] 2> 3095602 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40979.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e6c76b2
[junit4] 2> 3095603 INFO (jetty-launcher-6901-thread-1) [n:127.0.0.1:40979_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_FF4EC3AE66AAE64A-001/tempDir-001/node1/.
[junit4] 2> 3095646 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62976.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e6c76b2
[junit4] 2> 3095656 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62976.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e6c76b2
[junit4] 2> 3095656 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_62976.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e6c76b2
[junit4] 2> 3095657 INFO (jetty-launcher-6901-thread-2) [n:127.0.0.1:62976_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_FF4EC3AE66AAE64A-001/tempDir-001/node2/.
[junit4] 2> 3095683 INFO (zkConnectionManagerCallback-6926-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3095690 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160927999a20009, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3095795 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
[junit4] 2> 3095803 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidProxyUser-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
[junit4] 2> 3095892 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidRemote
[junit4] 2> 3095901 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidRemote-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidRemote
[junit4] 2> 3095968 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyUnknownRemote
[junit4] 2> 3106387 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyUnknownRemote-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyUnknownRemote
[junit4] 2> 3106508 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
[junit4] 2> 3106511 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigGroups-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
[junit4] 2> 3106570 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testForwarding
[junit4] 2> 3106578 INFO (zkConnectionManagerCallback-6929-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3107493 INFO (zkConnectionManagerCallback-6932-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3107495 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 3107501 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:49813/solr ready
[junit4] 2> 3107504 INFO (qtp65522055-30869) [n:127.0.0.1:62976_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3107508 INFO (OverseerThreadFactory-11736-thread-1-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.CreateCollectionCmd Create collection forwardingCollection
[junit4] 2> 3107630 INFO (OverseerStateUpdate-99240242768379910-127.0.0.1:62976_solr-n_0000000000) [n:127.0.0.1:62976_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"forwardingCollection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"forwardingCollection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:40979/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 3107833 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 3107834 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3107941 INFO (zkCallback-6917-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3107941 INFO (zkCallback-6917-thread-2-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3108855 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 3108878 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema [forwardingCollection_shard1_replica_n1] Schema name=test
[junit4] 2> 3108995 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema test/1.0 with uniqueid field id
[junit4] 2> 3109034 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'forwardingCollection_shard1_replica_n1' using configuration from collection forwardingCollection, trusted=true
[junit4] 2> 3109034 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_40979.solr.core.forwardingCollection.shard1.replica_n1' (registry 'solr.core.forwardingCollection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1e6c76b2
[junit4] 2> 3109034 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 3109034 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [[forwardingCollection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_FF4EC3AE66AAE64A-001/tempDir-001/node1/forwardingCollection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithSecureImpersonation_FF4EC3AE66AAE64A-001/tempDir-001/node1/./forwardingCollection_shard1_replica_n1/data/]
[junit4] 2> 3109036 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=30, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 3109094 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 3109094 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 3109095 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 3109095 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 3109095 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
[junit4] 2> 3109096 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@24c005dc[forwardingCollection_shard1_replica_n1] main]
[junit4] 2> 3109098 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
[junit4] 2> 3109102 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
[junit4] 2> 3109121 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 3109121 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587843899075854336
[junit4] 2> 3109121 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
[junit4] 2> 3109124 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
[junit4] 2> 3109124 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
[junit4] 2> 3109124 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
[junit4] 2> 3109124 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
[junit4] 2> 3109125 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
[junit4] 2> 3109125 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
[junit4] 2> 3109126 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
[junit4] 2> 3109126 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
[junit4] 2> 3109127 INFO (searcherExecutor-11741-thread-1-processing-n:127.0.0.1:40979_solr x:forwardingCollection_shard1_replica_n1 s:shard1 c:forwardingCollection r:core_node2) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] Registered new searcher Searcher@24c005dc[forwardingCollection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 3109129 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 3109129 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 3109129 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:40979/solr/forwardingCollection_shard1_replica_n1/
[junit4] 2> 3109129 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
[junit4] 2> 3109129 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SyncStrategy http://127.0.0.1:40979/solr/forwardingCollection_shard1_replica_n1/ has no replicas
[junit4] 2> 3109129 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 3109136 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:40979/solr/forwardingCollection_shard1_replica_n1/ shard1
[junit4] 2> 3109239 INFO (zkCallback-6917-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3109239 INFO (zkCallback-6917-thread-2-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3109289 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 3109296 INFO (qtp1896841339-30863) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node2&collection.configName=conf1&newCollection=true&name=forwardingCollection_shard1_replica_n1&action=CREATE&numShards=1&collection=forwardingCollection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=1462
[junit4] 2> 3109307 INFO (qtp65522055-30869) [n:127.0.0.1:62976_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 3109403 INFO (zkCallback-6917-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3109403 INFO (zkCallback-6917-thread-2-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/forwardingCollection/state.json] for collection [forwardingCollection] has occurred - updating... (live nodes size: [2])
[junit4] 2> 3109515 INFO (OverseerCollectionConfigSetProcessor-99240242768379910-127.0.0.1:62976_solr-n_0000000000) [n:127.0.0.1:62976_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 3110309 INFO (qtp65522055-30869) [n:127.0.0.1:62976_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={pullReplicas=0&replicationFactor=1&collection.configName=conf1&maxShardsPerNode=1&name=forwardingCollection&nrtReplicas=1&action=CREATE&numShards=1&tlogReplicas=0&wt=javabin&version=2} status=0 QTime=2805
[junit4] 2> 3110310 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.c.AbstractDistribZkTestBase Wait for recoveries to finish - collection: forwardingCollection failOnTimeout:true timeout (sec):100
[junit4] 2> 3110311 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.c.AbstractDistribZkTestBase Recoveries finished - collection: forwardingCollection
[junit4] 2> 3110315 INFO (qtp1896841339-30864) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=0
[junit4] 2> 3110325 INFO (qtp65522055-30868) [n:127.0.0.1:62976_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 3110331 INFO (qtp1896841339-30857) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.S.Request [forwardingCollection_shard1_replica_n1] webapp=/solr path=/select params={q=*:*&wt=javabin&version=2} hits=0 status=0 QTime=3
[junit4] 2> 3110332 INFO (TEST-TestSolrCloudWithSecureImpersonation.testForwarding-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testForwarding
[junit4] 2> 3110419 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
[junit4] 2> 3110494 WARN (qtp1896841339-30860) [ ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
[junit4] 2> id: invalid user name: "bar"
[junit4] 2>
[junit4] 2> 3110496 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyInvalidGroup-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
[junit4] 2> 3110686 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
[junit4] 2> 3110700 INFO (qtp1896841339-30861) [ ] o.a.s.s.SolrDispatchFilter Error authenticating
[junit4] 2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3110701 WARN (qtp1896841339-30861) [ ] o.e.j.s.ServletHandler /solr/admin/collections
[junit4] 2> org.apache.solr.common.SolrException: Error during request authentication,
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamDelegationTokenAuthenticationHandler$HttpParamAuthenticationHandler.authenticate(HttpParamDelegationTokenPlugin.java:213)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.DelegationTokenKerberosFilter.doFilter(DelegationTokenKerberosFilter.java:134)
[junit4] 2> at org.apache.solr.security.HttpParamDelegationTokenPlugin$HttpParamToRequestFilter.doFilter(HttpParamDelegationTokenPlugin.java:245)
[junit4] 2> at org.apache.solr.security.KerberosPlugin.doAuthenticate(KerberosPlugin.java:243)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> ... 26 more
[junit4] 2> 3110711 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNullProxyUser-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
[junit4] 2> 3110798 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
[junit4] 2> 3110842 INFO (qtp1896841339-30861) [n:127.0.0.1:40979_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3110842 INFO (qtp1896841339-30861) [n:127.0.0.1:40979_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3110845 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateGroup-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
[junit4] 2> 3111021 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
[junit4] 2> 3126074 WARN (jetty-launcher-6901-thread-1-SendThread(127.0.0.1:49813)) [n:127.0.0.1:40979_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 16670ms for sessionid 0x160927999a20005
[junit4] 2> 3126074 WARN (jetty-launcher-6901-thread-2-SendThread(127.0.0.1:49813)) [n:127.0.0.1:62976_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 19689ms for sessionid 0x160927999a20008
[junit4] 2> 3126074 WARN (jetty-launcher-6901-thread-1-SendThread(127.0.0.1:49813)) [n:127.0.0.1:40979_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 19689ms for sessionid 0x160927999a20007
[junit4] 2> 3126075 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160927999a20007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3126075 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160927999a20008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3126075 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160927999a20005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3126074 WARN (jetty-launcher-6901-thread-2-SendThread(127.0.0.1:49813)) [n:127.0.0.1:62976_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15748ms for sessionid 0x160927999a20006
[junit4] 2> 3126075 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160927999a20006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3126075 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyWrongHost-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
[junit4] 2> 3126147 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
[junit4] 2> 3126148 INFO (qtp1896841339-30864) [n:127.0.0.1:40979_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3126148 INFO (qtp1896841339-30864) [n:127.0.0.1:40979_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3126156 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateAnyHostAnyUser-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
[junit4] 2> 3126231 WARN (zkConnectionManagerCallback-6920-thread-1-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4d72a09c name: ZooKeeperConnection Watcher:127.0.0.1:49813/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3126231 WARN (zkConnectionManagerCallback-6920-thread-1-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3126232 WARN (zkConnectionManagerCallback-6932-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4ab3322 name: ZooKeeperConnection Watcher:127.0.0.1:49813/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3126236 WARN (zkConnectionManagerCallback-6932-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3126236 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
[junit4] 2> 3126239 WARN (zkConnectionManagerCallback-6918-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@516aaea8 name: ZooKeeperConnection Watcher:127.0.0.1:49813/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3126239 WARN (zkConnectionManagerCallback-6918-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3126262 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyNoConfigHosts-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
[junit4] 2> 3126373 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
[junit4] 2> 3126374 INFO (qtp1896841339-30857) [n:127.0.0.1:40979_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 3126374 INFO (qtp1896841339-30857) [n:127.0.0.1:40979_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 3126374 INFO (TEST-TestSolrCloudWithSecureImpersonation.testProxyValidateHost-seed#[FF4EC3AE66AAE64A]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
[junit4] 2> 3126890 INFO (jetty-closer-6902-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@78519b61{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3126891 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=94764562
[junit4] 2> 3126892 INFO (jetty-closer-6902-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@79f7cdfc{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 3126893 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3126893 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@55c9ceb8: rootName = solr_40979, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@1c872e47
[junit4] 2> 3126893 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=499534110
[junit4] 2> 3126894 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3126894 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3092b061: rootName = solr_62976, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@6a2d52b9
[junit4] 2> 3126903 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3126903 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4d6a1e70: rootName = solr_62976, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5ced0c29
[junit4] 2> 3126908 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3126908 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@651a99a2: rootName = solr_62976, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@3002c92
[junit4] 2> 3126909 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 3126909 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4fa2cc0a: rootName = solr_40979, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@5ced0c29
[junit4] 2> 3126913 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 3126913 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@bc3254b: rootName = solr_40979, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@3002c92
[junit4] 2> 3127607 INFO (zkConnectionManagerCallback-6918-thread-1-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3127611 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:40979_solr
[junit4] 2> 3127613 INFO (jetty-closer-6902-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3127613 INFO (zkCallback-6917-thread-8-processing-n:127.0.0.1:40979_solr) [n:127.0.0.1:40979_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 3127613 INFO (coreCloseExecutor-11746-thread-1) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.c.SolrCore [forwardingCollection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@d574410
[junit4] 2> 3127614 INFO (coreCloseExecutor-11746-thread-1) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.forwardingCollection.shard1.replica_n1, tag=223822864
[junit4] 2> 3127614 INFO (coreCloseExecutor-11746-thread-1) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@17e60777: rootName = solr_40979, domain = solr.core.forwardingCollection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.forwardingCollection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@c00d5b6
[junit4] 2> 3127625 INFO (coreCloseExecutor-11746-thread-1) [n:127.0.0.1:40979_solr c:forwardingCollection s:shard1 r:core_node2 x:forwardingCollection_shard1_replica_n1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.forwardingCollection.shard1.leader, tag=223822864
[junit4] 2> 3127635 ERROR (Thread[Thread-8006,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:40979_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 3127693 INFO (zkConnectionManagerCallback-6920-thread-1-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3127694 INFO (zkCallback-6919-thread-8-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 3127696 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:62976_solr
[junit4] 2> 3127698 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 3127698 INFO (jetty-closer-6902-thread-2) [ ] o.a.s.c.Overseer Overseer (id=99240242768379910-127.0.0.1:62976_solr-n_0000000000) closing
[junit4] 2> 3127698 INFO (OverseerStateUpdate-99240242768379910-127.0.0.1:62976_solr-n_0000000000) [n:127.0.0.1:62976_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:62976_solr
[junit4] 2> 3127699 INFO (zkCallback-6919-thread-8-processing-n:127.0.0.1:62976_solr) [n:127.0.0.1:62976_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 3127702 WARN (OverseerAutoScalingTriggerThread-99240242768379910-127.0.0.1:62976_solr-n_0000000000) [n:127.0.0.1:62976_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 3127707 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x160927999a20006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3127709 ERROR (Thread[Thread-8008,5,TGRP-TestSolrCloudWithSecureImpersonation]) [n:127.0.0.1:62976_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 3128042 WARN (jetty-launcher-6901-thread-1-SendThread(127.0.0.1:49813)) [n:127.0.0.1:40979_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x160927999a20007 has expired
[junit4] 2> 3128111 WARN (jetty-launcher-6901-thread-2-SendThread(127.0.0.1:49813)) [n:127.0.0.1:62976_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x160927999a20008 has expired
[junit4] 2> 3128144 INFO (jetty-closer-6902-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@5af889ba{/solr,null,UNAVAILABLE}
[junit4] 2> 3128212 INFO (jetty-closer-6902-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@30bbc5a9{/solr,null,UNAVAILABLE}
[junit4] 2> 3128214 ERROR (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3128215 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49813 49813
[junit4] 2> 3133269 INFO (Thread-7989) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:49813 49813
[junit4] 2> 3133271 WARN (Thread-7989) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/collections/forwardingCollection/state.json
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 3133305 INFO (SUITE-TestSolrCloudWithSecureImpersonation-seed#[FF4EC3AE66AAE64A]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Dec 26, 2017 10:59:29 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> Dec 26, 2017 10:59:49 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] 2> 1) Thread[id=30907, name=jetty-launcher-6901-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 2) Thread[id=30916, name=jetty-launcher-6901-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Dec 26, 2017 10:59:49 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=30907, name=jetty-launcher-6901-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> 2) Thread[id=30916, name=jetty-launcher-6901-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] 2> 3153498 ERROR (jetty-launcher-6901-thread-1-EventThread) [n:127.0.0.1:40979_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 3153500 ERROR (jetty-launcher-6901-thread-2-EventThread) [n:127.0.0.1:62976_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Dec 26, 2017 10:59:49 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70), sim=RandomSimilarity(queryNorm=true): {}, locale=hi, timezone=Asia/Yangon
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=64393424,total=522190848
[junit4] 2> NOTE: All tests run in this JVM: [PeerSyncReplicationTest, TestLegacyTerms, TestManagedResourceStorage, UninvertDocValuesMergePolicyTest, FullHLLTest, DeleteInactiveReplicaTest, HLLUtilTest, TestGenericDistributedQueue, SolrCmdDistributorTest, ResponseLogComponentTest, HdfsBasicDistributedZk2Test, TestTolerantUpdateProcessorRandomCloud, SpatialHeatmapFacetsTest, DisMaxRequestHandlerTest, RollingRestartTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TimeZoneUtilsTest, TestFilteredDocIdSet, TestInitParams, TestSizeLimitedDistributedMap, DistanceFunctionTest, NodeAddedTriggerTest, CloneFieldUpdateProcessorFactoryTest, UUIDFieldTest, ShowFileRequestHandlerTest, CircularListTest, ConfigSetsAPITest, ZkCLITest, TestTlogReplica, CdcrBidirectionalTest, TestMissingGroups, TestIBSimilarityFactory, SolrInfoBeanTest, TestHalfAndHalfDocValues, TestSerializedLuceneMatchVersion, TestRawTransformer, ClusterStateUpdateTest, HdfsCollectionsAPIDistributedZkTest, TestRangeQuery, PreAnalyzedUpdateProcessorTest, TestStressRecovery, TestPushWriter, ShardRoutingTest, MoveReplicaHDFSFailoverTest, SearchRateTriggerTest, OverseerTest, TestConfigSetImmutable, BigEndianAscendingWordDeserializerTest, TestUnifiedSolrHighlighter, TestDelegationWithHadoopAuth, TestQuerySenderListener, ConvertedLegacyTest, CollectionTooManyReplicasTest, TestSortByMinMaxFunction, SuggesterFSTTest, MinimalSchemaTest, IndexSchemaTest, TestFileDictionaryLookup, DefaultValueUpdateProcessorTest, ReplicationFactorTest, SpatialFilterTest, RecoveryZkTest, TestHighlightDedupGrouping, HdfsDirectoryFactoryTest, FastVectorHighlighterTest, JSONWriterTest, DistributedFacetPivotSmallAdvancedTest, TestCSVResponseWriter, TestPHPSerializedResponseWriter, TestPerFieldSimilarityWithDefaultOverride, TestXIncludeConfig, SolrJmxReporterCloudTest, TestNumericTokenStream, OutputWriterTest, UniqFieldsUpdateProcessorFactoryTest, TestFieldCollectionResource, TestSolrConfigHandlerCloud, TestSweetSpotSimilarityFactory, StatsReloadRaceTest, TestCloudPseudoReturnFields, LeaderFailureAfterFreshStartTest, HdfsRecoverLeaseTest, SuggestComponentTest, TestDistribStateManager, TestRandomRequestDistribution, BitVectorTest, HdfsThreadLeakTest, TestEmbeddedSolrServerSchemaAPI, BasicAuthStandaloneTest, TestBulkSchemaConcurrent, SolrCLIZkUtilsTest, TestFieldCacheWithThreads, PathHierarchyTokenizerFactoryTest, SaslZkACLProviderTest, TestSolr4Spatial, RankQueryTest, BlockJoinFacetDistribTest, TestWriterPerf, TestHdfsBackupRestoreCore, CoreMergeIndexesAdminHandlerTest, AddReplicaTest, TestConfigReload, SolrCoreMetricManagerTest, LeaderInitiatedRecoveryOnShardRestartTest, CoreAdminCreateDiscoverTest, MetricsHandlerTest, CursorPagingTest, SolrShardReporterTest, DataDrivenBlockJoinTest, DateMathParserTest, TestRandomDVFaceting, SolrXmlInZkTest, TestRecovery, OverseerModifyCollectionTest, TestHighFrequencyDictionaryFactory, TestFiltering, TestTriggerIntegration, TestSolrQueryResponse, ConfigureRecoveryStrategyTest, TestSolrCoreProperties, TestPointFields, TestSchemaSimilarityResource, DeleteNodeTest, ScriptEngineTest, TestBinaryResponseWriter, TestCustomStream, TestDeleteCollectionOnDownNodes, TestSearcherReuse, SliceStateTest, SignatureUpdateProcessorFactoryTest, URLClassifyProcessorTest, TestNumericRangeQuery32, DirectoryFactoryTest, VMParamsZkACLAndCredentialsProvidersTest, TemplateUpdateProcessorTest, AsyncCallRequestStatusResponseTest, TestEmbeddedSolrServerAdminHandler, TestUpdate, DistributedFacetPivotWhiteBoxTest, DateFieldTest, TestGraphMLResponseWriter, TestDownShardTolerantSearch, QueryParsingTest, TestCloudNestedDocsSort, DistributedQueryComponentOptimizationTest, HdfsChaosMonkeyNothingIsSafeTest, TestFuzzyAnalyzedSuggestions, MoveReplicaTest, TestOmitPositions, TestComplexPhraseLeadingWildcard, TestPhraseSuggestions, TestDistributedSearch, CheckHdfsIndexTest, TestCodecSupport, TestFieldCacheReopen, UUIDUpdateProcessorFallbackTest, SuggesterTSTTest, HdfsUnloadDistributedZkTest, TestCursorMarkWithoutUniqueKey, TestExactSharedStatsCache, TestSolrCloudWithHadoopAuthPlugin, FacetPivotSmallTest, LargeFieldTest, TestSubQueryTransformerDistrib, TestHdfsCloudBackupRestore, ZkControllerTest, SearchHandlerTest, ClassificationUpdateProcessorIntegrationTest, TestPseudoReturnFields, JavabinLoaderTest, TestPullReplicaErrorHandling, CSVRequestHandlerTest, SyncSliceTest, DistributedTermsComponentTest, DistribCursorPagingTest, RAMDirectoryFactoryTest, RulesTest, NodeMutatorTest, TestLocalFSCloudBackupRestore, SortByFunctionTest, CdcrRequestHandlerTest, TestCustomSort, TestPolicyCloud, TestSolrCLIRunExample, SuggesterWFSTTest, PrimUtilsTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, TestRandomFaceting, LeaderElectionTest, TestDistributedGrouping, TestRealTimeGet, TestStressReorder, TestCoreContainer, StatsComponentTest, BadIndexSchemaTest, TestSort, TestFunctionQuery, TestLazyCores, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, TestCoreDiscovery, SpellCheckCollatorTest, CoreAdminHandlerTest, SolrRequestParserTest, TestFoldingMultitermQuery, SuggesterTest, PolyFieldTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, DocValuesMultiTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, TestQueryUtils, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, TermsComponentTest, XmlUpdateRequestHandlerTest, RequestHandlersTest, DocumentBuilderTest, TermVectorComponentTest, TestIndexingPerformance, MoreLikeThisHandlerTest, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, RegexBoostProcessorTest, SolrPluginUtilsTest, TestJmxIntegration, TestCollationField, ReturnFieldsTest, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestLFUCache, HighlighterConfigTest, UpdateParamsTest, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, MultiTermTest, TestDocSet, TestBinaryField, TestSearchPerf, NumericFieldsTest, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestCharFilters, SynonymTokenizerTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestBM25SimilarityFactory, TestFastWriter, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, ChaosMonkeyNothingIsSafeTest, ChaosMonkeyNothingIsSafeWithPullReplicasTest, CleanupOldIndexTest, CollectionStateFormat2Test, ConnectionManagerTest, CreateCollectionCleanupTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, DocValuesNotIndexedTest, ForceLeaderTest, MigrateRouteKeyTest, OverriddenZkACLAndCredentialsProvidersTest, RemoteQueryErrorTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, TestCloudJSONFacetJoinDomain, TestCloudSearcherWarming, TestCollectionAPI, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDistribDocBasedVersion, TestLeaderElectionZkExpiry, TestPrepRecovery, TestRandomFlRTGCloud, TestReplicaProperties, TestRequestStatusCollectionAPI, TestSegmentSorting, TestSolrCloudWithDelegationTokens, TestSolrCloudWithSecureImpersonation]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestSolrCloudWithSecureImpersonation -Dtests.seed=FF4EC3AE66AAE64A -Dtests.slow=true -Dtests.locale=hi -Dtests.timezone=Asia/Yangon -Dtests.asserts=true -Dtests.file.encoding=UTF-8
[junit4] ERROR 0.00s J0 | TestSolrCloudWithSecureImpersonation (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.TestSolrCloudWithSecureImpersonation:
[junit4] > 1) Thread[id=30907, name=jetty-launcher-6901-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > 2) Thread[id=30916, name=jetty-launcher-6901-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestSolrCloudWithSecureImpersonation]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > at __randomizedtesting.SeedInfo.seed([FF4EC3AE66AAE64A]:0)
[junit4] Completed [561/764 (1!)] on J0 in 58.54s, 12 tests, 1 error <<< FAILURES!
[...truncated 48829 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 352 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/352/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseParallelGC
2 tests failed.
FAILED: org.apache.solr.cloud.autoscaling.AutoAddReplicasPlanActionTest.testSimple
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:57248/solr]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:57248/solr]
at __randomizedtesting.SeedInfo.seed([8C36CBF943959C44:B485EF0764664895]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
at org.apache.solr.cloud.autoscaling.AutoAddReplicasPlanActionTest.testSimple(AutoAddReplicasPlanActionTest.java:110)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57248/solr
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:657)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
... 44 more
Caused by: java.net.SocketException: Connection reset
at java.net.SocketInputStream.read(SocketInputStream.java:210)
at java.net.SocketInputStream.read(SocketInputStream.java:141)
at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:137)
at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:153)
at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:282)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:138)
at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:56)
at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:259)
at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:163)
at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:165)
at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:273)
at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:125)
at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:272)
at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:185)
at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:89)
at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:111)
at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:185)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:542)
... 48 more
FAILED: org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI
Error Message:
Error from server at http://127.0.0.1:65204/solr/awhollynewcollection_0: No registered leader was found after waiting for 4000ms , collection: awhollynewcollection_0 slice: shard1 saw state=DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/6)={ "pullReplicas":"0", "replicationFactor":"3", "shards":{"shard1":{ "range":"80000000-7fffffff", "state":"active", "replicas":{ "core_node3":{ "core":"awhollynewcollection_0_shard1_replica_n1", "base_url":"http://127.0.0.1:65204/solr", "node_name":"127.0.0.1:65204_solr", "state":"active", "type":"NRT"}, "core_node5":{ "core":"awhollynewcollection_0_shard1_replica_n2", "base_url":"http://127.0.0.1:36577/solr", "node_name":"127.0.0.1:36577_solr", "state":"down", "type":"NRT"}, "core_node6":{ "core":"awhollynewcollection_0_shard1_replica_n4", "base_url":"http://127.0.0.1:51613/solr", "node_name":"127.0.0.1:51613_solr", "state":"active", "type":"NRT"}}}}, "router":{"name":"compositeId"}, "maxShardsPerNode":"1", "autoAddReplicas":"false", "nrtReplicas":"3", "tlogReplicas":"0"} with live_nodes=[127.0.0.1:51891_solr, 127.0.0.1:36577_solr, 127.0.0.1:65204_solr, 127.0.0.1:51613_solr]
Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:65204/solr/awhollynewcollection_0: No registered leader was found after waiting for 4000ms , collection: awhollynewcollection_0 slice: shard1 saw state=DocCollection(awhollynewcollection_0//collections/awhollynewcollection_0/state.json/6)={
"pullReplicas":"0",
"replicationFactor":"3",
"shards":{"shard1":{
"range":"80000000-7fffffff",
"state":"active",
"replicas":{
"core_node3":{
"core":"awhollynewcollection_0_shard1_replica_n1",
"base_url":"http://127.0.0.1:65204/solr",
"node_name":"127.0.0.1:65204_solr",
"state":"active",
"type":"NRT"},
"core_node5":{
"core":"awhollynewcollection_0_shard1_replica_n2",
"base_url":"http://127.0.0.1:36577/solr",
"node_name":"127.0.0.1:36577_solr",
"state":"down",
"type":"NRT"},
"core_node6":{
"core":"awhollynewcollection_0_shard1_replica_n4",
"base_url":"http://127.0.0.1:51613/solr",
"node_name":"127.0.0.1:51613_solr",
"state":"active",
"type":"NRT"}}}},
"router":{"name":"compositeId"},
"maxShardsPerNode":"1",
"autoAddReplicas":"false",
"nrtReplicas":"3",
"tlogReplicas":"0"} with live_nodes=[127.0.0.1:51891_solr, 127.0.0.1:36577_solr, 127.0.0.1:65204_solr, 127.0.0.1:51613_solr]
at __randomizedtesting.SeedInfo.seed([8C36CBF943959C44:C443BF4D45A6B3D1]:0)
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:413)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.request.UpdateRequest.commit(UpdateRequest.java:233)
at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:460)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Build Log:
[...truncated 12946 lines...]
[junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
[junit4] 2> 2353815 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/init-core-data-001
[junit4] 2> 2353816 WARN (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=53 numCloses=53
[junit4] 2> 2353817 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 2353820 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2353822 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001
[junit4] 2> 2353822 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2353824 INFO (Thread-5206) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2353824 INFO (Thread-5206) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2353827 ERROR (Thread-5206) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2353922 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:33125
[junit4] 2> 2353927 INFO (zkConnectionManagerCallback-3912-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2353947 INFO (jetty-launcher-3909-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2353947 INFO (jetty-launcher-3909-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2353950 INFO (jetty-launcher-3909-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2353950 INFO (jetty-launcher-3909-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5f580040{/solr,null,AVAILABLE}
[junit4] 2> 2353953 INFO (jetty-launcher-3909-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@488e54f6{/solr,null,AVAILABLE}
[junit4] 2> 2353953 INFO (jetty-launcher-3909-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6cecadc8{HTTP/1.1,[http/1.1]}{127.0.0.1:51613}
[junit4] 2> 2353953 INFO (jetty-launcher-3909-thread-3) [ ] o.e.j.s.Server Started @2357752ms
[junit4] 2> 2353954 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51613}
[junit4] 2> 2353954 ERROR (jetty-launcher-3909-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2353954 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2353954 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2353954 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2353954 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T23:32:47.482Z
[junit4] 2> 2353956 INFO (jetty-launcher-3909-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5a6c2723{/solr,null,AVAILABLE}
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6ef721db{HTTP/1.1,[http/1.1]}{127.0.0.1:65204}
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-4) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-1) [ ] o.e.j.s.Server Started @2357755ms
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=65204}
[junit4] 2> 2353957 ERROR (jetty-launcher-3909-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2353957 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T23:32:47.485Z
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@496a45a0{/solr,null,AVAILABLE}
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@6c3a8445{HTTP/1.1,[http/1.1]}{127.0.0.1:51891}
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.e.j.s.Server Started @2357756ms
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=51891}
[junit4] 2> 2353958 ERROR (jetty-launcher-3909-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2353958 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T23:32:47.486Z
[junit4] 2> 2353969 INFO (jetty-launcher-3909-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@52efc4ed{HTTP/1.1,[http/1.1]}{127.0.0.1:36577}
[junit4] 2> 2353969 INFO (jetty-launcher-3909-thread-2) [ ] o.e.j.s.Server Started @2357767ms
[junit4] 2> 2353969 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=36577}
[junit4] 2> 2353970 ERROR (jetty-launcher-3909-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2353970 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2353970 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2353970 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2353970 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T23:32:47.498Z
[junit4] 2> 2353977 INFO (zkConnectionManagerCallback-3916-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2353978 INFO (zkConnectionManagerCallback-3914-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2353979 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2353979 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2353980 INFO (zkConnectionManagerCallback-3920-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2353980 INFO (zkConnectionManagerCallback-3918-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2353983 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2353985 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2353988 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2353988 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2353988 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2353989 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2354011 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2354011 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2354018 INFO (jetty-launcher-3909-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 2354023 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverWorkLoopDelay is ignored
[junit4] 2> 2354028 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.c.SolrXmlConfig Configuration parameter autoReplicaFailoverBadNodeExpiration is ignored
[junit4] 2> 2354029 INFO (jetty-launcher-3909-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 2354029 INFO (zkConnectionManagerCallback-3924-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354034 INFO (jetty-launcher-3909-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 2354035 INFO (zkConnectionManagerCallback-3928-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354041 INFO (zkConnectionManagerCallback-3932-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354042 INFO (zkConnectionManagerCallback-3934-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354045 INFO (jetty-launcher-3909-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:33125/solr
[junit4] 2> 2354058 INFO (zkConnectionManagerCallback-3940-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354065 INFO (zkConnectionManagerCallback-3936-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354066 INFO (zkConnectionManagerCallback-3942-thread-1-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354067 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x16090057e7b0009, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2354070 INFO (zkConnectionManagerCallback-3944-thread-1-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354409 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2354410 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:51891_solr
[junit4] 2> 2354417 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.c.Overseer Overseer (id=99237545073377292-127.0.0.1:51891_solr-n_0000000000) starting
[junit4] 2> 2354441 INFO (jetty-launcher-3909-thread-2) [n:127.0.0.1:36577_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2354442 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51891_solr
[junit4] 2> 2354452 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2354453 INFO (OverseerStateUpdate-99237545073377292-127.0.0.1:51891_solr-n_0000000000) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2354459 INFO (jetty-launcher-3909-thread-1) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 2354463 INFO (jetty-launcher-3909-thread-2) [n:127.0.0.1:36577_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36577_solr
[junit4] 2> 2354465 INFO (jetty-launcher-3909-thread-3) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 2354466 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2354467 INFO (zkCallback-3943-thread-1-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2354470 INFO (jetty-launcher-3909-thread-3) [n:127.0.0.1:51613_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2354479 INFO (zkCallback-3933-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 2354484 INFO (jetty-launcher-3909-thread-3) [n:127.0.0.1:51613_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51613_solr
[junit4] 2> 2354502 INFO (zkCallback-3933-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2354502 INFO (jetty-launcher-3909-thread-1) [n:127.0.0.1:65204_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2354504 INFO (zkCallback-3941-thread-1-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2354505 INFO (zkCallback-3943-thread-2-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2354505 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 2354514 INFO (jetty-launcher-3909-thread-1) [n:127.0.0.1:65204_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:65204_solr
[junit4] 2> 2354589 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2354589 INFO (zkCallback-3943-thread-2-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2354596 INFO (zkCallback-3941-thread-1-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2354611 INFO (jetty-launcher-3909-thread-3) [n:127.0.0.1:51613_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354618 INFO (zkCallback-3933-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 2354638 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354662 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354662 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354664 INFO (jetty-launcher-3909-thread-3) [n:127.0.0.1:51613_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354664 INFO (jetty-launcher-3909-thread-3) [n:127.0.0.1:51613_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354669 INFO (jetty-launcher-3909-thread-1) [n:127.0.0.1:65204_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354670 INFO (jetty-launcher-3909-thread-4) [n:127.0.0.1:51891_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node4
[junit4] 2> 2354709 INFO (jetty-launcher-3909-thread-3) [n:127.0.0.1:51613_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node3
[junit4] 2> 2354729 INFO (jetty-launcher-3909-thread-1) [n:127.0.0.1:65204_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354729 INFO (jetty-launcher-3909-thread-1) [n:127.0.0.1:65204_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354734 INFO (jetty-launcher-3909-thread-1) [n:127.0.0.1:65204_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node2
[junit4] 2> 2354884 INFO (jetty-launcher-3909-thread-2) [n:127.0.0.1:36577_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354898 INFO (jetty-launcher-3909-thread-2) [n:127.0.0.1:36577_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354898 INFO (jetty-launcher-3909-thread-2) [n:127.0.0.1:36577_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2354901 INFO (jetty-launcher-3909-thread-2) [n:127.0.0.1:36577_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node1
[junit4] 2> 2354994 INFO (zkConnectionManagerCallback-3954-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2354999 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x16090057e7b000d, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2355000 INFO (zkConnectionManagerCallback-3958-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2355002 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2355004 INFO (SUITE-CollectionsAPIDistributedZkTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:33125/solr ready
[junit4] 2> 2355090 INFO (TEST-CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[8C36CBF943959C44]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testCoresAreDistributedAcrossNodes
[junit4] 2> 2355093 INFO (TEST-CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[8C36CBF943959C44]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2355107 INFO (qtp1940934753-20716) [n:127.0.0.1:51891_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=2&collection.configName=conf&name=nodes_used_collection&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2355116 INFO (OverseerThreadFactory-7092-thread-1-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.CreateCollectionCmd Create collection nodes_used_collection
[junit4] 2> 2355249 INFO (OverseerStateUpdate-99237545073377292-127.0.0.1:51891_solr-n_0000000000) [n:127.0.0.1:51891_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"nodes_used_collection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"nodes_used_collection_shard1_replica_n1",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:65204/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2355252 INFO (OverseerStateUpdate-99237545073377292-127.0.0.1:51891_solr-n_0000000000) [n:127.0.0.1:51891_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"nodes_used_collection",
[junit4] 2> "shard":"shard1",
[junit4] 2> "core":"nodes_used_collection_shard1_replica_n2",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:51613/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2355256 INFO (OverseerStateUpdate-99237545073377292-127.0.0.1:51891_solr-n_0000000000) [n:127.0.0.1:51891_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"nodes_used_collection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"nodes_used_collection_shard2_replica_n4",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:51891/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2355264 INFO (OverseerStateUpdate-99237545073377292-127.0.0.1:51891_solr-n_0000000000) [n:127.0.0.1:51891_solr ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"ADDREPLICA",
[junit4] 2> "collection":"nodes_used_collection",
[junit4] 2> "shard":"shard2",
[junit4] 2> "core":"nodes_used_collection_shard2_replica_n7",
[junit4] 2> "state":"down",
[junit4] 2> "base_url":"http://127.0.0.1:36577/solr",
[junit4] 2> "type":"NRT",
[junit4] 2> "waitForFinalState":"false"}
[junit4] 2> 2355480 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica_n7&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2355484 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica_n2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2355483 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica_n4&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2355494 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica_n1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT
[junit4] 2> 2355485 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2355495 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2355495 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2355495 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 2355621 INFO (zkCallback-3943-thread-1-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2355621 INFO (zkCallback-3935-thread-2-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2355622 INFO (zkCallback-3933-thread-2-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2355622 INFO (zkCallback-3943-thread-2-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2355622 INFO (zkCallback-3941-thread-2-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2355622 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2355622 INFO (zkCallback-3933-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2355624 INFO (zkCallback-3941-thread-1-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356522 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2356527 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2356538 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.s.IndexSchema [nodes_used_collection_shard2_replica_n7] Schema name=minimal
[junit4] 2> 2356538 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.s.IndexSchema [nodes_used_collection_shard1_replica_n1] Schema name=minimal
[junit4] 2> 2356542 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2356542 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard2_replica_n7' using configuration from collection nodes_used_collection, trusted=true
[junit4] 2> 2356543 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.nodes_used_collection.shard2.replica_n7' (registry 'solr.core.nodes_used_collection.shard2.replica_n7') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2356543 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2356543 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.SolrCore [[nodes_used_collection_shard2_replica_n7] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node1/nodes_used_collection_shard2_replica_n7], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node1/nodes_used_collection_shard2_replica_n7/data/]
[junit4] 2> 2356544 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2356544 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard1_replica_n1' using configuration from collection nodes_used_collection, trusted=true
[junit4] 2> 2356545 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.nodes_used_collection.shard1.replica_n1' (registry 'solr.core.nodes_used_collection.shard1.replica_n1') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2356545 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2356545 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.c.SolrCore [[nodes_used_collection_shard1_replica_n1] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node2/nodes_used_collection_shard1_replica_n1], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node2/nodes_used_collection_shard1_replica_n1/data/]
[junit4] 2> 2356562 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2356571 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.s.IndexSchema [nodes_used_collection_shard1_replica_n2] Schema name=minimal
[junit4] 2> 2356573 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2356574 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard1_replica_n2' using configuration from collection nodes_used_collection, trusted=true
[junit4] 2> 2356574 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.nodes_used_collection.shard1.replica_n2' (registry 'solr.core.nodes_used_collection.shard1.replica_n2') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2356574 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2356588 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.SolrCore [[nodes_used_collection_shard1_replica_n2] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node3/nodes_used_collection_shard1_replica_n2], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node3/nodes_used_collection_shard1_replica_n2/data/]
[junit4] 2> 2356608 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.3.0
[junit4] 2> 2356621 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.s.IndexSchema [nodes_used_collection_shard2_replica_n4] Schema name=minimal
[junit4] 2> 2356623 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
[junit4] 2> 2356623 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.c.CoreContainer Creating SolrCore 'nodes_used_collection_shard2_replica_n4' using configuration from collection nodes_used_collection, trusted=true
[junit4] 2> 2356624 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.core.nodes_used_collection.shard2.replica_n4' (registry 'solr.core.nodes_used_collection.shard2.replica_n4') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@64a4aa44
[junit4] 2> 2356624 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.c.SolrCore solr.RecoveryStrategy.Builder
[junit4] 2> 2356624 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.c.SolrCore [[nodes_used_collection_shard2_replica_n4] ] Opening new SolrCore at [/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node4/nodes_used_collection_shard2_replica_n4], dataDir=[/export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_8C36CBF943959C44-001/tempDir-001/node4/nodes_used_collection_shard2_replica_n4/data/]
[junit4] 2> 2356732 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2356732 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2356733 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2356733 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2356735 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.s.SolrIndexSearcher Opening [Searcher@79d20c7b[nodes_used_collection_shard1_replica_n2] main]
[junit4] 2> 2356738 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2356740 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2356741 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2356741 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587800724229586944
[junit4] 2> 2356747 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2356747 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2356748 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2356748 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2356750 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.s.SolrIndexSearcher Opening [Searcher@177bc616[nodes_used_collection_shard2_replica_n7] main]
[junit4] 2> 2356752 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2356752 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2356753 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2356753 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587800724242169856
[junit4] 2> 2356757 INFO (searcherExecutor-7103-thread-1-processing-n:127.0.0.1:51613_solr x:nodes_used_collection_shard1_replica_n2 s:shard1 c:nodes_used_collection r:core_node5) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica_n2] Registered new searcher Searcher@79d20c7b[nodes_used_collection_shard1_replica_n2] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2356773 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2356774 INFO (searcherExecutor-7101-thread-1-processing-n:127.0.0.1:36577_solr x:nodes_used_collection_shard2_replica_n7 s:shard2 c:nodes_used_collection r:core_node8) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica_n7] Registered new searcher Searcher@177bc616[nodes_used_collection_shard2_replica_n7] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2356783 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=9999ms
[junit4] 2> 2356793 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2356793 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2356795 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2356795 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2356800 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.s.SolrIndexSearcher Opening [Searcher@29e6e5c1[nodes_used_collection_shard2_replica_n4] main]
[junit4] 2> 2356802 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2356803 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2356804 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2356804 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587800724295647232
[junit4] 2> 2356809 INFO (searcherExecutor-7104-thread-1-processing-n:127.0.0.1:51891_solr x:nodes_used_collection_shard2_replica_n4 s:shard2 c:nodes_used_collection r:core_node6) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica_n4] Registered new searcher Searcher@29e6e5c1[nodes_used_collection_shard2_replica_n4] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2356841 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
[junit4] 2> 2356841 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir=null defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
[junit4] 2> 2356843 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
[junit4] 2> 2356843 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
[junit4] 2> 2356845 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3297a036[nodes_used_collection_shard1_replica_n1] main]
[junit4] 2> 2356854 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf
[junit4] 2> 2356855 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf
[junit4] 2> 2356855 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.h.ReplicationHandler Commits will be reserved for 10000ms.
[junit4] 2> 2356856 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1587800724350173184
[junit4] 2> 2356862 INFO (searcherExecutor-7102-thread-1-processing-n:127.0.0.1:65204_solr x:nodes_used_collection_shard1_replica_n1 s:shard1 c:nodes_used_collection r:core_node3) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica_n1] Registered new searcher Searcher@3297a036[nodes_used_collection_shard1_replica_n1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
[junit4] 2> 2356893 INFO (zkCallback-3933-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356893 INFO (zkCallback-3933-thread-2-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356893 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356894 INFO (zkCallback-3935-thread-2-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356894 INFO (zkCallback-3943-thread-2-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356895 INFO (zkCallback-3943-thread-3-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356896 INFO (zkCallback-3941-thread-1-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2356895 INFO (zkCallback-3941-thread-2-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357275 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2357275 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2357275 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:36577/solr/nodes_used_collection_shard2_replica_n7/
[junit4] 2> 2357276 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard2_replica_n7 url=http://127.0.0.1:36577/solr START replicas=[http://127.0.0.1:51891/solr/nodes_used_collection_shard2_replica_n4/] nUpdates=100
[junit4] 2> 2357280 INFO (qtp1940934753-20720) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.c.S.Request [nodes_used_collection_shard2_replica_n4] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2357284 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
[junit4] 2> 2357284 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
[junit4] 2> 2357284 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:51613/solr/nodes_used_collection_shard1_replica_n2/
[junit4] 2> 2357285 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard1_replica_n2 url=http://127.0.0.1:51613/solr START replicas=[http://127.0.0.1:65204/solr/nodes_used_collection_shard1_replica_n1/] nUpdates=100
[junit4] 2> 2357287 INFO (qtp904198077-20696) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.c.S.Request [nodes_used_collection_shard1_replica_n1] webapp=/solr path=/get params={distrib=false&qt=/get&fingerprint=false&getVersions=100&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 2357583 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard2_replica_n7 url=http://127.0.0.1:36577/solr DONE. We have no versions. sync failed.
[junit4] 2> 2357584 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2357584 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2357584 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2357591 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.u.PeerSync PeerSync: core=nodes_used_collection_shard1_replica_n2 url=http://127.0.0.1:51613/solr DONE. We have no versions. sync failed.
[junit4] 2> 2357591 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.SyncStrategy Leader's attempt to sync with shard failed, moving to the next candidate
[junit4] 2> 2357592 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
[junit4] 2> 2357592 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext Found all replicas participating in election, clear LIR
[junit4] 2> 2357606 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:36577/solr/nodes_used_collection_shard2_replica_n7/ shard2
[junit4] 2> 2357629 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.ShardLeaderElectionContext I am the new leader: http://127.0.0.1:51613/solr/nodes_used_collection_shard1_replica_n2/ shard1
[junit4] 2> 2357737 INFO (zkCallback-3941-thread-1-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357737 INFO (zkCallback-3941-thread-2-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357737 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357738 INFO (zkCallback-3935-thread-2-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357738 INFO (zkCallback-3933-thread-2-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357738 INFO (zkCallback-3933-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357738 INFO (zkCallback-3943-thread-3-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357739 INFO (zkCallback-3943-thread-2-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2357761 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2357772 INFO (qtp179557452-20688) [n:127.0.0.1:36577_solr c:nodes_used_collection s:shard2 r:core_node8 x:nodes_used_collection_shard2_replica_n7] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node8&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica_n7&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2294
[junit4] 2> 2357782 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.c.ZkController I am the leader, no recovery necessary
[junit4] 2> 2357794 INFO (qtp978399054-20705) [n:127.0.0.1:51613_solr c:nodes_used_collection s:shard1 r:core_node5 x:nodes_used_collection_shard1_replica_n2] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node5&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica_n2&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2310
[junit4] 2> 2357826 INFO (qtp1940934753-20717) [n:127.0.0.1:51891_solr c:nodes_used_collection s:shard2 r:core_node6 x:nodes_used_collection_shard2_replica_n4] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node6&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard2_replica_n4&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard2&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2347
[junit4] 2> 2357880 INFO (qtp904198077-20695) [n:127.0.0.1:65204_solr c:nodes_used_collection s:shard1 r:core_node3 x:nodes_used_collection_shard1_replica_n1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&coreNodeName=core_node3&collection.configName=conf&newCollection=true&name=nodes_used_collection_shard1_replica_n1&action=CREATE&numShards=2&collection=nodes_used_collection&shard=shard1&wt=javabin&version=2&replicaType=NRT} status=0 QTime=2387
[junit4] 2> 2357884 INFO (qtp1940934753-20716) [n:127.0.0.1:51891_solr ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
[junit4] 2> 2358009 INFO (zkCallback-3935-thread-2-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358009 INFO (zkCallback-3935-thread-1-processing-n:127.0.0.1:36577_solr) [n:127.0.0.1:36577_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358009 INFO (zkCallback-3943-thread-3-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358009 INFO (zkCallback-3943-thread-2-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358009 INFO (zkCallback-3933-thread-1-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358009 INFO (zkCallback-3933-thread-2-processing-n:127.0.0.1:65204_solr) [n:127.0.0.1:65204_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358009 INFO (zkCallback-3941-thread-2-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358009 INFO (zkCallback-3941-thread-1-processing-n:127.0.0.1:51613_solr) [n:127.0.0.1:51613_solr ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/nodes_used_collection/state.json] for collection [nodes_used_collection] has occurred - updating... (live nodes size: [4])
[junit4] 2> 2358885 INFO (qtp1940934753-20716) [n:127.0.0.1:51891_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=2&collection.configName=conf&name=nodes_used_collection&nrtReplicas=2&action=CREATE&numShards=2&wt=javabin&version=2} status=0 QTime=3779
[junit4] 2> 2358886 INFO (TEST-CollectionsAPIDistributedZkTest.testCoresAreDistributedAcrossNodes-seed#[8C36CBF943959C44]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testCoresAreDistributedAcrossNodes
[junit4] 2> 2358962 INFO (TEST-CollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[8C36CBF943959C44]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testDeleteNonExistentCollection
[junit4] 2> 2358968 INFO (TEST-CollectionsAPIDistributedZkTest.testDeleteNonExistentCollection-seed#[8C36CBF943959C44]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (4)
[junit4] 2> 2358971 INFO (qtp1940934753-20726) [n:127.0.0.1:51891_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=nodes_used_collection&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 2358976 INFO (OverseerCollectionConfigSetProcessor-99237545073377292-127.0.0.1:51891_solr-n_0000000000) [n:127.0.0.1:51891_solr ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist. Requestor may have disconnected from ZooKeeper
[junit4] 2> 2358977 INFO (OverseerThreadFactory-7092-thread-2-processing-n:127.0.0.1:51891_solr) [n:127.0.0.1:51891_solr ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
[junit4] 2> 2358978 INFO (qtp1940934753-20728) [n:127.0.0.1:51891_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.nodes_used_collection.shard2.replica_n4, tag=null
[junit4] 2> 2358978 INFO (qtp1940934753-20728) [n:127.0.0.1:51891_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@25c59c1: rootName = null, domain = solr.core.nodes_used_collection.shard2.replica_n4, service url = null, agent id = null] for registry solr.core.nodes_used_collection.shard2.replica_n4 / com.codahale.metrics.MetricRegistry@276f7822
[junit4] 2> 2358978 INFO (qtp179557452-20700) [n:127.0.0.1:36577_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.nodes_used_collection.shard2.replica_n7, tag=null
[junit4] 2> 2358979 INFO (qtp179557452-20700) [n:127.0.0.1:36577_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3f4ce826: rootName = null, domain = solr.core.nodes_used_collection.shard2.replica_n7, service url = null, agent id = null] for registry solr.core.nodes_used_collection.shard2.replica_n7 / com.codahale.metrics.MetricRegistry@19cb3b20
[junit4] 2> 2358979 INFO (qtp978399054-20707) [n:127.0.0.1:51613_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.nodes_used_collection.shard1.replica_n2, tag=null
[junit4] 2> 2358979 INFO (qtp978399054-20707) [n:127.0.0.1:51613_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@fff5b60: rootName = null, domain = solr.core.nodes_used_collection.shard1.replica_n2, service url = null, agent id = null] for registry solr.core.nodes_used_collection.shard1.replica_n2 / com.codahale.metrics.MetricRegistry@77192490
[junit4] 2> 2358981 INFO (qtp904198077-20692) [n:127.0.0.1:65204_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.nodes_used_collection.shard1.replica_n1, tag=null
[junit4] 2> 2358981 INFO (qtp904198077-20692) [n:127.0.0.1:65204_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@20ecd8a0: rootName = null, domain = solr.core.nodes_used_collection.shard1.replica_n1, service url = null, agent id = null] for registry solr.core.nodes_used_collection.shard1.replica_n1 / com.codahale.metrics.MetricRegistry@77402002
[junit4] 2> 2359119 INFO (qtp179557452-20700) [n:127.0.0.1:36577_solr ] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica_n7] CLOSING SolrCore org.apache.solr.core.SolrCore@78eb4e42
[junit4] 2> 2359124 INFO (qtp179557452-20700) [n:127.0.0.1:36577_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.nodes_used_collection.shard2.replica_n7, tag=2028686914
[junit4] 2> 2359129 INFO (qtp904198077-20692) [n:127.0.0.1:65204_solr ] o.a.s.c.SolrCore [nodes_used_collection_shard1_replica_n1] CLOSING SolrCore org.apache.solr.core.SolrCore@61122d6c
[junit4] 2> 2359132 INFO (qtp904198077-20692) [n:127.0.0.1:65204_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.nodes_used_collection.shard1.replica_n1, tag=1628581228
[junit4] 2> 2359132 INFO (qtp904198077-20692) [n:127.0.0.1:65204_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.nodes_used_collection.shard1.leader, tag=1628581228
[junit4] 2> 2359124 INFO (qtp1940934753-20728) [n:127.0.0.1:51891_solr ] o.a.s.c.SolrCore [nodes_used_collection_shard2_replica_n4] CLOSING SolrCore org.apache.solr.core.SolrCore@e1a359b
[junit4] 2> 2359132 INFO (qtp179557452-20700) [n:127.0.0.1:36577_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.nodes_used_collection.shard2.leader, tag=2028686914
[junit4] 2> 2359135 INFO (qtp1940934753-20728) [n:127.0.0.1:51891_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registr
[...truncated too long message...]
try=solr.collection.testSimple3.shard1.leader, tag=1809051209
[junit4] 2> 4041538 INFO (coreCloseExecutor-9428-thread-4) [n:127.0.0.1:35874_solr c:testSimple3 s:shard3 r:core_node6 x:testSimple3_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.testSimple3.shard3.replica_n4, tag=1569902508
[junit4] 2> 4041539 INFO (coreCloseExecutor-9428-thread-4) [n:127.0.0.1:35874_solr c:testSimple3 s:shard3 r:core_node6 x:testSimple3_shard3_replica_n4] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@44f77963: rootName = solr_35874, domain = solr.core.testSimple3.shard3.replica_n4, service url = null, agent id = null] for registry solr.core.testSimple3.shard3.replica_n4 / com.codahale.metrics.MetricRegistry@7006d623
[junit4] 2> 4041544 INFO (coreCloseExecutor-9428-thread-4) [n:127.0.0.1:35874_solr c:testSimple3 s:shard3 r:core_node6 x:testSimple3_shard3_replica_n4] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.collection.testSimple3.shard3.leader, tag=1569902508
[junit4] 2> 4041545 INFO (jetty-closer-6369-thread-1) [ ] o.a.s.c.Overseer Overseer (id=99237654805086215-127.0.0.1:35874_solr-n_0000000001) closing
[junit4] 2> 4041545 INFO (OverseerStateUpdate-99237654805086215-127.0.0.1:35874_solr-n_0000000001) [n:127.0.0.1:35874_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:35874_solr
[junit4] 2> 4041545 WARN (OverseerAutoScalingTriggerThread-99237654805086215-127.0.0.1:35874_solr-n_0000000001) [n:127.0.0.1:35874_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 4041557 INFO (jetty-closer-6369-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@61a29342{/solr,null,UNAVAILABLE}
[junit4] 2> 4041559 ERROR (SUITE-AutoAddReplicasPlanActionTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 4041560 INFO (SUITE-AutoAddReplicasPlanActionTest-seed#[8C36CBF943959C44]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:61103 61103
[junit4] 2> 4051695 INFO (Thread-8017) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:61103 61103
[junit4] 2> 4051700 WARN (Thread-8017) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/aliases.json
[junit4] 2> 5 /solr/clusterprops.json
[junit4] 2> 4 /solr/security.json
[junit4] 2> 4 /solr/configs/conf
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 26 /solr/collections/testSimple1/state.json
[junit4] 2> 25 /solr/collections/testSimple2/state.json
[junit4] 2> 6 /solr/collections/testSimple3/state.json
[junit4] 2> 5 /solr/clusterstate.json
[junit4] 2> 2 /solr/collections/testSimple1/leader_elect/shard1/election/99237654805086215-core_node3-n_0000000000
[junit4] 2> 2 /solr/autoscaling.json
[junit4] 2> 2 /solr/collections/testSimple1/leader_elect/shard2/election/99237654805086215-core_node7-n_0000000000
[junit4] 2> 2 /solr/collections/testSimple2/leader_elect/shard1/election/99237654805086217-core_node4-n_0000000000
[junit4] 2> 2 /solr/overseer_elect/election/99237654805086215-127.0.0.1:35874_solr-n_0000000001
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 5 /solr/live_nodes
[junit4] 2> 5 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/autoscaling/events/.auto_add_replicas
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.cloud.autoscaling.AutoAddReplicasPlanActionTest_8C36CBF943959C44-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=61, maxMBSortInHeap=7.587713011722901, sim=RandomSimilarity(queryNorm=false): {}, locale=sq, timezone=Asia/Barnaul
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=188147232,total=507510784
[junit4] 2> NOTE: All tests run in this JVM: [WordBreakSolrSpellCheckerTest, TestLocalFSCloudBackupRestore, TestDFRSimilarityFactory, ShardRoutingTest, TestMiniSolrCloudClusterSSL, TestManagedSynonymGraphFilterFactory, SuggesterFSTTest, TestGenericDistributedQueue, TestUseDocValuesAsStored2, BadIndexSchemaTest, TestCollectionAPIs, TestStandardQParsers, ReturnFieldsTest, SyncSliceTest, TestLeaderElectionZkExpiry, TestPolicyCloud, TestGraphMLResponseWriter, ConvertedLegacyTest, IndexSchemaTest, SolrXmlInZkTest, TestInPlaceUpdatesDistrib, UUIDUpdateProcessorFallbackTest, SpellCheckCollatorTest, ClassificationUpdateProcessorIntegrationTest, TestFieldCacheWithThreads, ShowFileRequestHandlerTest, TestLegacyTerms, ExecutePlanActionTest, TestDFISimilarityFactory, ZkCLITest, OverriddenZkACLAndCredentialsProvidersTest, UUIDFieldTest, TestLRUStatsCache, RegexBoostProcessorTest, TestSystemCollAutoCreate, DirectUpdateHandlerTest, DefaultValueUpdateProcessorTest, TestAddFieldRealTimeGet, XmlUpdateRequestHandlerTest, TestRecovery, TestRequestStatusCollectionAPI, TestFieldCacheSortRandom, TestSegmentSorting, TestCollectionAPI, ManagedSchemaRoundRobinCloudTest, HLLUtilTest, TestAtomicUpdateErrorCases, CdcrBidirectionalTest, TestPrepRecovery, URLClassifyProcessorTest, DateFieldTest, TestQueryWrapperFilter, TestIBSimilarityFactory, SignatureUpdateProcessorFactoryTest, TestSolr4Spatial, UtilsToolTest, TestReversedWildcardFilterFactory, PrimUtilsTest, HttpTriggerListenerTest, TestExactSharedStatsCache, SolrInfoBeanTest, TestUtils, RemoteQueryErrorTest, AutoAddReplicasIntegrationTest, RollingRestartTest, TestInitParams, CircularListTest, BlockJoinFacetSimpleTest, TemplateUpdateProcessorTest, NodeAddedTriggerTest, SolrTestCaseJ4Test, OverseerModifyCollectionTest, TimeRoutedAliasUpdateProcessorTest, TestUninvertingReader, SolrSlf4jReporterTest, TestDeleteCollectionOnDownNodes, TestUnifiedSolrHighlighter, TestTlogReplica, ReplicationFactorTest, HdfsRecoverLeaseTest, ConfigSetsAPITest, DataDrivenBlockJoinTest, TestImplicitCoreProperties, TestOmitPositions, HdfsThreadLeakTest, QueryParsingTest, TestBackupRepositoryFactory, TestSchemaSimilarityResource, TestSchemalessBufferedUpdates, TestSolrDeletionPolicy2, TestSortByMinMaxFunction, DisMaxRequestHandlerTest, TestCodecSupport, RecoveryZkTest, DistributedQueryComponentOptimizationTest, TestCloudDeleteByQuery, TestHdfsBackupRestoreCore, TestTolerantUpdateProcessorRandomCloud, XsltUpdateRequestHandlerTest, PrimitiveFieldTypeTest, PeerSyncReplicationTest, HttpSolrCallGetCoreTest, TestConfigSetsAPIZkFailure, SortByFunctionTest, TestDynamicFieldCollectionResource, OverseerTest, RankQueryTest, TestSolrXml, TestReloadAndDeleteDocs, JSONWriterTest, TestStressInPlaceUpdates, DocValuesMissingTest, TestCustomSort, TestNumericTokenStream, FullHLLTest, CloneFieldUpdateProcessorFactoryTest, MoveReplicaHDFSFailoverTest, DeleteShardTest, HdfsCollectionsAPIDistributedZkTest, TestSolrCoreSnapshots, LeaderElectionTest, HdfsChaosMonkeyNothingIsSafeTest, MoveReplicaTest, TestRandomDVFaceting, DistanceFunctionTest, TestLegacyNumericRangeQueryBuilder, TestAnalyzeInfixSuggestions, PathHierarchyTokenizerFactoryTest, MultiTermTest, TestHdfsCloudBackupRestore, TestPullReplicaErrorHandling, StressHdfsTest, TestCoreAdminApis, HdfsRestartWhileUpdatingTest, SuggesterTest, TestFieldTypeCollectionResource, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestPushWriter, CdcrReplicationDistributedZkTest, TestRangeQuery, TestPointFields, TestNodeAddedTrigger, TestSolrCloudWithDelegationTokens, DirectoryFactoryTest, RAMDirectoryFactoryTest, CurrencyFieldTypeTest, DistributedFacetPivotWhiteBoxTest, TestMinMaxOnMultiValuedField, OverseerStatusTest, SolrJmxReporterCloudTest, SpatialFilterTest, ChaosMonkeySafeLeaderWithPullReplicasTest, TestTriggerIntegration, TestSweetSpotSimilarityFactory, MetricsHandlerTest, CSVRequestHandlerTest, TestReqParamsAPI, TestSolrCoreProperties, TestDistributedSearch, CdcrRequestHandlerTest, AlternateDirectoryTest, DistributedQueryElevationComponentTest, TestDistribStateManager, TestSolrCoreParser, TestDownShardTolerantSearch, IndexBasedSpellCheckerTest, ClusterStateUpdateTest, TestReplicationHandler, TestFuzzyAnalyzedSuggestions, StatsComponentTest, TestIndexingPerformance, TestJsonFacetRefinement, SuggestComponentTest, TestRawTransformer, TestBulkSchemaConcurrent, CoreMergeIndexesAdminHandlerTest, DistributedFacetPivotSmallAdvancedTest, TestPolicyCloud, TestBinaryField, TestLMDirichletSimilarityFactory, BasicDistributedZkTest, TestManagedResourceStorage, TestNumericTerms32, TestCursorMarkWithoutUniqueKey, TestSearcherReuse, TestReplicaProperties, AssignTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, CreateCollectionCleanupTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DistributedVersionInfoTest, DocValuesNotIndexedTest, ForceLeaderTest, LeaderInitiatedRecoveryOnCommitTest, MoveReplicaHDFSTest, TestCloudPseudoReturnFields, TestClusterProperties, TestCollectionsAPIViaSolrCloudCluster, TestConfigSetsAPIExclusivity, TestDistribDocBasedVersion, TestRandomRequestDistribution, TestRequestForwarding, TestSizeLimitedDistributedMap, TestStressLiveNodes, VMParamsZkACLAndCredentialsProvidersTest, AutoAddReplicasPlanActionTest]
[junit4] Completed [567/764 (2!)] on J0 in 23.65s, 1 test, 1 error <<< FAILURES!
[...truncated 48715 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 351 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/351/
Java: 64bit/jdk1.8.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.NodeLostTriggerTest
Error Message:
java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:55906 within 30000 ms
Stack Trace:
org.apache.solr.common.SolrException: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:55906 within 30000 ms
at __randomizedtesting.SeedInfo.seed([2D8766AA3E232ACF]:0)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:182)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:119)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:114)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:101)
at org.apache.solr.cloud.MiniSolrCloudCluster.waitForAllNodes(MiniSolrCloudCluster.java:268)
at org.apache.solr.cloud.MiniSolrCloudCluster.<init>(MiniSolrCloudCluster.java:262)
at org.apache.solr.cloud.SolrCloudTestCase$Builder.configure(SolrCloudTestCase.java:190)
at org.apache.solr.cloud.autoscaling.NodeLostTriggerTest.setupCluster(NodeLostTriggerTest.java:60)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:874)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.TimeoutException: Could not connect to ZooKeeper 127.0.0.1:55906 within 30000 ms
at org.apache.solr.common.cloud.ConnectionManager.waitForConnected(ConnectionManager.java:232)
at org.apache.solr.common.cloud.SolrZkClient.<init>(SolrZkClient.java:174)
... 31 more
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.autoscaling.NodeLostTriggerTest
Error Message:
61 threads leaked from SUITE scope at org.apache.solr.cloud.autoscaling.NodeLostTriggerTest: 1) Thread[id=3042, name=jetty-launcher-703-thread-2, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2) Thread[id=3075, name=qtp1650845160-3075, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 3) Thread[id=3047, name=qtp1650845160-3047, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 4) Thread[id=3087, name=qtp1729671336-3087, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 5) Thread[id=3045, name=jetty-launcher-703-thread-5, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 6) Thread[id=3060, name=qtp1591435603-3060, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 7) Thread[id=3061, name=qtp1591435603-3061-acceptor-0@40e8e7d9-ServerConnector@46b714e5{SSL,[ssl, http/1.1]}{127.0.0.1:52597}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 8) Thread[id=3048, name=qtp1650845160-3048, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 9) Thread[id=3064, name=qtp1740481082-3064-acceptor-0@4c1faa66-ServerConnector@b0b0354{SSL,[ssl, http/1.1]}{127.0.0.1:55906}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 10) Thread[id=3090, name=qtp1729671336-3090, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 11) Thread[id=3041, name=jetty-launcher-703-thread-1, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 12) Thread[id=3054, name=org.eclipse.jetty.server.session.HashSessionManager@3977c11eTimer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 13) Thread[id=3057, name=qtp1591435603-3057, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 14) Thread[id=3085, name=qtp1740481082-3085, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 15) Thread[id=3037, name=ProcessThread(sid:0 cport:55906):, state=WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122) 16) Thread[id=3089, name=qtp1729671336-3089-acceptor-0@398090ef-ServerConnector@1ab96779{SSL,[ssl, http/1.1]}{127.0.0.1:58169}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 17) Thread[id=3091, name=qtp1729671336-3091, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 18) Thread[id=3050, name=qtp1650845160-3050-acceptor-0@36d356ce-ServerConnector@5a1303e{SSL,[ssl, http/1.1]}{127.0.0.1:39615}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 19) Thread[id=3094, name=qtp1729671336-3094, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 20) Thread[id=3086, name=qtp1740481082-3086, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 21) Thread[id=3074, name=qtp1650845160-3074, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 22) Thread[id=3071, name=qtp1740481082-3071, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 23) Thread[id=3062, name=qtp1591435603-3062, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 24) Thread[id=3096, name=qtp1591435603-3096, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 25) Thread[id=3103, name=qtp580786915-3103, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 26) Thread[id=3063, name=org.eclipse.jetty.server.session.HashSessionManager@56dd660dTimer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 27) Thread[id=3102, name=qtp580786915-3102, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 28) Thread[id=3097, name=qtp580786915-3097, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 29) Thread[id=3044, name=jetty-launcher-703-thread-4, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 30) Thread[id=3108, name=Scheduler-675764870, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 31) Thread[id=3093, name=qtp1729671336-3093, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 32) Thread[id=3078, name=qtp1591435603-3078, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 33) Thread[id=3084, name=qtp1740481082-3084, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 34) Thread[id=3098, name=qtp580786915-3098-acceptor-0@6a4c823-ServerConnector@f72f22{SSL,[ssl, http/1.1]}{127.0.0.1:64999}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422) at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250) at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371) at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 35) Thread[id=3070, name=qtp1740481082-3070, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 36) Thread[id=3095, name=org.eclipse.jetty.server.session.HashSessionManager@d7da345Timer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 37) Thread[id=3056, name=qtp1591435603-3056, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 38) Thread[id=3069, name=qtp1740481082-3069, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 39) Thread[id=3053, name=qtp1650845160-3053, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 40) Thread[id=3107, name=org.eclipse.jetty.server.session.HashSessionManager@640fa2e3Timer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 41) Thread[id=3106, name=qtp580786915-3106, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 42) Thread[id=3049, name=qtp1650845160-3049, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 43) Thread[id=3088, name=qtp1729671336-3088, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 44) Thread[id=3100, name=qtp580786915-3100, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 45) Thread[id=3073, name=qtp1650845160-3073, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 46) Thread[id=3046, name=qtp1650845160-3046, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 47) Thread[id=3052, name=qtp1650845160-3052, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 48) Thread[id=3072, name=org.eclipse.jetty.server.session.HashSessionManager@4ac53bf6Timer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 49) Thread[id=3092, name=qtp1729671336-3092, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 50) Thread[id=3059, name=qtp1591435603-3059, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 51) Thread[id=3066, name=qtp1740481082-3066, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 52) Thread[id=3034, name=NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:173) at java.lang.Thread.run(Thread.java:748) 53) Thread[id=3068, name=qtp1740481082-3068, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 54) Thread[id=3099, name=qtp580786915-3099, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 55) Thread[id=3105, name=qtp580786915-3105, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 56) Thread[id=3035, name=SessionTracker, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at java.lang.Object.wait(Native Method) at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:146) 57) Thread[id=3043, name=jetty-launcher-703-thread-3, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 58) Thread[id=3033, name=Thread-584, state=WAITING, group=TGRP-NodeLostTriggerTest] at java.lang.Object.wait(Native Method) at java.lang.Thread.join(Thread.java:1252) at java.lang.Thread.join(Thread.java:1326) at org.apache.zookeeper.server.NIOServerCnxnFactory.join(NIOServerCnxnFactory.java:297) at org.apache.solr.cloud.ZkTestServer$ZKServerMain.runFromConfig(ZkTestServer.java:310) at org.apache.solr.cloud.ZkTestServer$2.run(ZkTestServer.java:491) 59) Thread[id=3058, name=qtp1591435603-3058, state=RUNNABLE, group=TGRP-NodeLostTriggerTest] at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method) at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223) at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243) at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148) at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100) at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147) at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589) at java.lang.Thread.run(Thread.java:748) 60) Thread[id=3065, name=qtp1740481082-3065, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748) 61) Thread[id=3081, name=qtp1591435603-3081, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392) at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563) at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48) at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 61 threads leaked from SUITE scope at org.apache.solr.cloud.autoscaling.NodeLostTriggerTest:
1) Thread[id=3042, name=jetty-launcher-703-thread-2, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
2) Thread[id=3075, name=qtp1650845160-3075, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
3) Thread[id=3047, name=qtp1650845160-3047, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
4) Thread[id=3087, name=qtp1729671336-3087, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
5) Thread[id=3045, name=jetty-launcher-703-thread-5, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
6) Thread[id=3060, name=qtp1591435603-3060, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
7) Thread[id=3061, name=qtp1591435603-3061-acceptor-0@40e8e7d9-ServerConnector@46b714e5{SSL,[ssl, http/1.1]}{127.0.0.1:52597}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
8) Thread[id=3048, name=qtp1650845160-3048, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
9) Thread[id=3064, name=qtp1740481082-3064-acceptor-0@4c1faa66-ServerConnector@b0b0354{SSL,[ssl, http/1.1]}{127.0.0.1:55906}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
10) Thread[id=3090, name=qtp1729671336-3090, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
11) Thread[id=3041, name=jetty-launcher-703-thread-1, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
12) Thread[id=3054, name=org.eclipse.jetty.server.session.HashSessionManager@3977c11eTimer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
13) Thread[id=3057, name=qtp1591435603-3057, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
14) Thread[id=3085, name=qtp1740481082-3085, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
15) Thread[id=3037, name=ProcessThread(sid:0 cport:55906):, state=WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
at org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:122)
16) Thread[id=3089, name=qtp1729671336-3089-acceptor-0@398090ef-ServerConnector@1ab96779{SSL,[ssl, http/1.1]}{127.0.0.1:58169}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
17) Thread[id=3091, name=qtp1729671336-3091, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
18) Thread[id=3050, name=qtp1650845160-3050-acceptor-0@36d356ce-ServerConnector@5a1303e{SSL,[ssl, http/1.1]}{127.0.0.1:39615}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
19) Thread[id=3094, name=qtp1729671336-3094, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
20) Thread[id=3086, name=qtp1740481082-3086, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
21) Thread[id=3074, name=qtp1650845160-3074, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
22) Thread[id=3071, name=qtp1740481082-3071, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
23) Thread[id=3062, name=qtp1591435603-3062, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
24) Thread[id=3096, name=qtp1591435603-3096, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
25) Thread[id=3103, name=qtp580786915-3103, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
26) Thread[id=3063, name=org.eclipse.jetty.server.session.HashSessionManager@56dd660dTimer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
27) Thread[id=3102, name=qtp580786915-3102, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
28) Thread[id=3097, name=qtp580786915-3097, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
29) Thread[id=3044, name=jetty-launcher-703-thread-4, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
30) Thread[id=3108, name=Scheduler-675764870, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
31) Thread[id=3093, name=qtp1729671336-3093, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
32) Thread[id=3078, name=qtp1591435603-3078, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
33) Thread[id=3084, name=qtp1740481082-3084, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
34) Thread[id=3098, name=qtp580786915-3098-acceptor-0@6a4c823-ServerConnector@f72f22{SSL,[ssl, http/1.1]}{127.0.0.1:64999}, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:371)
at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:601)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
35) Thread[id=3070, name=qtp1740481082-3070, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
36) Thread[id=3095, name=org.eclipse.jetty.server.session.HashSessionManager@d7da345Timer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
37) Thread[id=3056, name=qtp1591435603-3056, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
38) Thread[id=3069, name=qtp1740481082-3069, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
at java.lang.Thread.run(Thread.java:748)
39) Thread[id=3053, name=qtp1650845160-3053, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:563)
at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:48)
at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:626)
at java.lang.Thread.run(Thread.java:748)
40) Thread[id=3107, name=org.eclipse.jetty.server.session.HashSessionManager@640fa2e3Timer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
41) Thread[id=3106, name=qtp580786915-3106, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:
[...truncated too long message...]
r.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 10) Thread[id=3054, name=org.eclipse.jetty.server.session.HashSessionManager@3977c11eTimer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 11) Thread[id=3063, name=org.eclipse.jetty.server.session.HashSessionManager@56dd660dTimer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 12) Thread[id=3107, name=org.eclipse.jetty.server.session.HashSessionManager@640fa2e3Timer, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 13) Thread[id=3097, name=qtp580786915-3097, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.nio.ch.IOUtil.drain(Native Method)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:109)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 14) Thread[id=3044, name=jetty-launcher-703-thread-4, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 15) Thread[id=3108, name=Scheduler-675764870, state=WAITING, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1081)
[junit4] > at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 16) Thread[id=3043, name=jetty-launcher-703-thread-3, state=TIMED_WAITING, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
[junit4] > at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
[junit4] > at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073)
[junit4] > at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
[junit4] > at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > 17) Thread[id=3058, name=qtp1591435603-3058, state=RUNNABLE, group=TGRP-NodeLostTriggerTest]
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll0(Native Method)
[junit4] > at sun.nio.ch.DevPollArrayWrapper.poll(DevPollArrayWrapper.java:223)
[junit4] > at sun.nio.ch.DevPollSelectorImpl.doSelect(DevPollSelectorImpl.java:98)
[junit4] > at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
[junit4] > at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:243)
[junit4] > at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:191)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:249)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] > at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:100)
[junit4] > at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:147)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] > at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([2D8766AA3E232ACF]:0)
[junit4] Completed [47/764 (1!)] on J1 in 87.00s, 0 tests, 3 errors <<< FAILURES!
[...truncated 50481 lines...]
[JENKINS] Lucene-Solr-7.x-Solaris (64bit/jdk1.8.0) - Build # 350 -
Still Unstable!
Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: https://jenkins.thetaphi.de/job/Lucene-Solr-7.x-Solaris/350/
Java: 64bit/jdk1.8.0 -XX:-UseCompressedOops -XX:+UseG1GC
3 tests failed.
FAILED: org.apache.solr.cloud.MoveReplicaHDFSTest.testFailedMove
Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:64692/solr/MoveReplicaHDFSTest_failed_coll_true, http://127.0.0.1:34789/solr/MoveReplicaHDFSTest_failed_coll_true]
Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:64692/solr/MoveReplicaHDFSTest_failed_coll_true, http://127.0.0.1:34789/solr/MoveReplicaHDFSTest_failed_coll_true]
at __randomizedtesting.SeedInfo.seed([BA3454F2C111D1B0:10F9870076C20460]:0)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:462)
at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1104)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:884)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:991)
at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:991)
at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:817)
at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:194)
at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
at org.apache.solr.cloud.MoveReplicaTest.testFailedMove(MoveReplicaTest.java:306)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34789/solr/MoveReplicaHDFSTest_failed_coll_true: no servers hosting shard: shard1
at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:643)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:255)
at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:244)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:483)
at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:436)
... 47 more
FAILED: org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication
Error Message:
Index: 0, Size: 0
Stack Trace:
java.lang.IndexOutOfBoundsException: Index: 0, Size: 0
at __randomizedtesting.SeedInfo.seed([BA3454F2C111D1B0:AE7C0FA7E2166CAE]:0)
at java.util.ArrayList.rangeCheck(ArrayList.java:657)
at java.util.ArrayList.get(ArrayList.java:433)
at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:561)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:934)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:970)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:984)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:817)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:468)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:943)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:829)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:879)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:890)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth
Error Message:
2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth: 1) Thread[id=4051, name=jetty-launcher-665-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505) 2) Thread[id=4055, name=jetty-launcher-665-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth] at sun.misc.Unsafe.park(Native Method) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323) at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105) at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279) at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41) at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244) at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44) at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61) at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
1) Thread[id=4051, name=jetty-launcher-665-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
2) Thread[id=4055, name=jetty-launcher-665-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
at sun.misc.Unsafe.park(Native Method)
at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
at __randomizedtesting.SeedInfo.seed([BA3454F2C111D1B0]:0)
Build Log:
[...truncated 11929 lines...]
[junit4] Suite: org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth
[junit4] 2> 624269 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_BA3454F2C111D1B0-001/init-core-data-001
[junit4] 2> 624270 WARN (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=5 numCloses=5
[junit4] 2> 624270 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 624272 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 624312 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 2 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_BA3454F2C111D1B0-001/tempDir-001
[junit4] 2> 624312 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 624313 INFO (Thread-786) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 624313 INFO (Thread-786) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 624315 ERROR (Thread-786) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 624413 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:50390
[junit4] 2> 624418 INFO (zkConnectionManagerCallback-668-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 624424 INFO (jetty-launcher-665-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 624424 INFO (jetty-launcher-665-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 624426 INFO (jetty-launcher-665-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@2454d891{/solr,null,AVAILABLE}
[junit4] 2> 624426 INFO (jetty-launcher-665-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@22ebd6a2{/solr,null,AVAILABLE}
[junit4] 2> 624427 INFO (jetty-launcher-665-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4b8d3166{HTTP/1.1,[http/1.1]}{127.0.0.1:57655}
[junit4] 2> 624427 INFO (jetty-launcher-665-thread-1) [ ] o.e.j.s.Server Started @627158ms
[junit4] 2> 624427 INFO (jetty-launcher-665-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=57655}
[junit4] 2> 624428 ERROR (jetty-launcher-665-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 624429 INFO (jetty-launcher-665-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 624429 INFO (jetty-launcher-665-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 624429 INFO (jetty-launcher-665-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 624429 INFO (jetty-launcher-665-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T00:12:41.110Z
[junit4] 2> 624431 INFO (jetty-launcher-665-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@3e5962dc{HTTP/1.1,[http/1.1]}{127.0.0.1:38227}
[junit4] 2> 624431 INFO (jetty-launcher-665-thread-2) [ ] o.e.j.s.Server Started @627162ms
[junit4] 2> 624431 INFO (jetty-launcher-665-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=38227}
[junit4] 2> 624431 ERROR (jetty-launcher-665-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 624431 INFO (jetty-launcher-665-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 624431 INFO (jetty-launcher-665-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 624431 INFO (jetty-launcher-665-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 624431 INFO (jetty-launcher-665-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T00:12:41.112Z
[junit4] 2> 624434 INFO (zkConnectionManagerCallback-670-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 624436 INFO (zkConnectionManagerCallback-672-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 624436 INFO (jetty-launcher-665-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 624442 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1608b03a8a30001, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 624442 INFO (jetty-launcher-665-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 624460 INFO (jetty-launcher-665-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50390/solr
[junit4] 2> 624460 INFO (jetty-launcher-665-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:50390/solr
[junit4] 2> 624464 INFO (zkConnectionManagerCallback-680-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 624465 INFO (zkConnectionManagerCallback-678-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 624470 INFO (zkConnectionManagerCallback-682-thread-1-processing-n:127.0.0.1:38227_solr) [n:127.0.0.1:38227_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 624473 INFO (zkConnectionManagerCallback-684-thread-1-processing-n:127.0.0.1:57655_solr) [n:127.0.0.1:57655_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 624510 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 624511 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57655_solr
[junit4] 2> 624517 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.Overseer Overseer (id=99232039632568326-127.0.0.1:57655_solr-n_0000000000) starting
[junit4] 2> 624524 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57655_solr
[junit4] 2> 624528 INFO (OverseerStateUpdate-99232039632568326-127.0.0.1:57655_solr-n_0000000000) [n:127.0.0.1:57655_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 624571 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
[junit4] 2> 624582 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=staff, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
[junit4] 2> 624621 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 624623 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 624625 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38227_solr
[junit4] 2> 624635 INFO (zkCallback-681-thread-1-processing-n:127.0.0.1:38227_solr) [n:127.0.0.1:38227_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 624654 INFO (zkCallback-683-thread-2-processing-n:127.0.0.1:57655_solr) [n:127.0.0.1:57655_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 624687 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 624719 INFO (jetty-launcher-665-thread-1-EventThread) [n:127.0.0.1:57655_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 624723 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.c.CoreContainer Initializing authentication plugin: org.apache.solr.security.HadoopAuthPlugin
[junit4] 2> 624723 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.s.HadoopAuthPlugin Params: {proxyuser.anyHostUsersGroup.hosts=*, proxyuser.wrongHost.hosts=1.1.1.1.1.1, proxyuser.bogusGroup.groups=__some_bogus_group, proxyuser.localHostAnyGroup.groups=*, cookie.domain=127.0.0.1, proxyuser.wrongHost.groups=*, type=simple, proxyuser.noHosts.groups=*, proxyuser.anyHostUsersGroup.groups=staff, proxyuser.noGroups.hosts=*, proxyuser.anyHostAnyUser.groups=*, delegation-token.token-kind=solr-dt, proxyuser.bogusGroup.hosts=*, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, signer.secret.provider.zookeeper.path=/token, zk-dt-secret-manager.enable=true, proxyuser.localHostAnyGroup.hosts=serv1-vm3,localhost,127.0.0.1, proxyuser.anyHostAnyUser.hosts=*, signer.secret.provider=zookeeper}
[junit4] 2> 624743 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.c.f.i.CuratorFrameworkImpl Starting
[junit4] 2> 624751 INFO (jetty-launcher-665-thread-2-EventThread) [n:127.0.0.1:38227_solr ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
[junit4] 2> 624870 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
[junit4] 2> 624886 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.CoreContainer PKIAuthenticationPlugin is managing internode requests
[junit4] 2> 624915 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38227.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@757e0b06
[junit4] 2> 624920 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38227.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@757e0b06
[junit4] 2> 624920 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_38227.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@757e0b06
[junit4] 2> 624921 INFO (jetty-launcher-665-thread-2) [n:127.0.0.1:38227_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_BA3454F2C111D1B0-001/tempDir-001/node2/.
[junit4] 2> 624960 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57655.solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@757e0b06
[junit4] 2> 624966 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57655.solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@757e0b06
[junit4] 2> 624966 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr_57655.solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@757e0b06
[junit4] 2> 624967 INFO (jetty-launcher-665-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.security.hadoop.TestImpersonationWithHadoopAuth_BA3454F2C111D1B0-001/tempDir-001/node1/.
[junit4] 2> 624996 INFO (zkConnectionManagerCallback-690-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625015 INFO (zkConnectionManagerCallback-694-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 625027 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 625031 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:50390/solr ready
[junit4] 2> 625054 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidProxyUser
[junit4] 2> 625094 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bar&p=bar&t=simple-dt&e=1514196761745&s=VkZc1KXDcImMHrPltowCCyVWtLM="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:41 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:41 GMT
[junit4] 2> 625095 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidProxyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidProxyUser
[junit4] 2> 625109 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigGroups
[junit4] 2> 625116 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noGroups&p=noGroups&t=simple-dt&e=1514196761793&s=97CjrePR+B8oLHVQcy4sbc/XeKg="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:41 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:41 GMT
[junit4] 2> 625117 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigGroups-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigGroups
[junit4] 2> 625144 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateGroup
[junit4] 2> 625190 INFO (qtp214175776-4002) [n:127.0.0.1:38227_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 625191 INFO (qtp214175776-4002) [n:127.0.0.1:38227_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=jenkins&action=LIST&user.name=anyHostUsersGroup&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 625191 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostUsersGroup&p=anyHostUsersGroup&t=simple-dt&e=1514196761868&s=vjGNwOUspdqk9QWtpbhE0LHwaV4="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:41 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:41 GMT
[junit4] 2> 625191 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateGroup-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateGroup
[junit4] 2> 625205 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateAnyHostAnyUser
[junit4] 2> 625207 INFO (qtp214175776-4006) [n:127.0.0.1:38227_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 625207 INFO (qtp214175776-4006) [n:127.0.0.1:38227_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=anyHostAnyUser&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 625208 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=anyHostAnyUser&p=anyHostAnyUser&t=simple-dt&e=1514196761887&s=SKfuGylVkaGRweGIl13X7qLpnzo="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:41 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:41 GMT
[junit4] 2> 625208 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateAnyHostAnyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateAnyHostAnyUser
[junit4] 2> 625222 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyInvalidGroup
[junit4] 2> 625244 WARN (qtp214175776-4003) [ ] o.a.h.s.ShellBasedUnixGroupsMapping got exception trying to get groups for user bar: id: invalid user name: "bar"
[junit4] 2> id: invalid user name: "bar"
[junit4] 2>
[junit4] 2> 625246 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=bogusGroup&p=bogusGroup&t=simple-dt&e=1514196761904&s=2yEkKO0XOZeleNe2+SQp8PQb3jg="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:41 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:41 GMT
[junit4] 2> 625247 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyInvalidGroup-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyInvalidGroup
[junit4] 2> 625270 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNoConfigHosts
[junit4] 2> 625279 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=noHosts&p=noHosts&t=simple-dt&e=1514196761955&s=secj4yoVwxN4sBeHOr85nm04M/E="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:41 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:41 GMT
[junit4] 2> 625281 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNoConfigHosts-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNoConfigHosts
[junit4] IGNOR/A 0.00s J1 | TestImpersonationWithHadoopAuth.testForwarding
[junit4] > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/HADOOP-9893))
[junit4] 2> 625301 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyNullProxyUser
[junit4] 2> 625305 INFO (qtp214175776-4004) [ ] o.a.s.s.SolrDispatchFilter Error authenticating
[junit4] 2> java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
[junit4] 2> at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 625305 WARN (qtp214175776-4004) [ ] o.e.j.s.ServletHandler /solr/admin/collections
[junit4] 2> org.apache.solr.common.SolrException: Error during request authentication,
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:457)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:347)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:326)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:139)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)
[junit4] 2> at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)
[junit4] 2> at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
[junit4] 2> at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)
[junit4] 2> at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
[junit4] 2> at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)
[junit4] 2> at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
[junit4] 2> at org.eclipse.jetty.server.Server.handle(Server.java:534)
[junit4] 2> at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:320)
[junit4] 2> at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)
[junit4] 2> at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)
[junit4] 2> at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)
[junit4] 2> at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)
[junit4] 2> at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)
[junit4] 2> at org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: java.lang.IllegalArgumentException: userName is NULL, empty or contains a '&'
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.checkForIllegalArgument(AuthToken.java:87)
[junit4] 2> at org.apache.hadoop.security.authentication.util.AuthToken.<init>(AuthToken.java:70)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationToken.<init>(AuthenticationToken.java:58)
[junit4] 2> at org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler.authenticate(PseudoAuthenticationHandler.java:195)
[junit4] 2> at org.apache.hadoop.security.token.delegation.web.DelegationTokenAuthenticationHandler.authenticate(DelegationTokenAuthenticationHandler.java:348)
[junit4] 2> at org.apache.solr.security.RequestContinuesRecorderAuthenticationHandler.authenticate(RequestContinuesRecorderAuthenticationHandler.java:69)
[junit4] 2> at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:518)
[junit4] 2> at org.apache.solr.security.HadoopAuthFilter.doFilter(HadoopAuthFilter.java:110)
[junit4] 2> at org.apache.solr.security.HadoopAuthPlugin.doAuthenticate(HadoopAuthPlugin.java:245)
[junit4] 2> at org.apache.solr.servlet.SolrDispatchFilter.authenticateRequest(SolrDispatchFilter.java:451)
[junit4] 2> ... 26 more
[junit4] 2> 625307 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyNullProxyUser-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyNullProxyUser
[junit4] 2> 625323 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyWrongHost
[junit4] 2> 640360 WARN (jetty-launcher-665-thread-1-SendThread(127.0.0.1:50390)) [n:127.0.0.1:57655_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15326ms for sessionid 0x1608b03a8a30007
[junit4] 2> 640360 WARN (jetty-launcher-665-thread-2-SendThread(127.0.0.1:50390)) [n:127.0.0.1:38227_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15331ms for sessionid 0x1608b03a8a30008
[junit4] 2> 640360 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1608b03a8a30007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 640361 WARN (jetty-launcher-665-thread-2-SendThread(127.0.0.1:50390)) [n:127.0.0.1:38227_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15438ms for sessionid 0x1608b03a8a30005
[junit4] 2> 640361 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1608b03a8a30008, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 640361 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1608b03a8a30005, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 640361 WARN (jetty-launcher-665-thread-1-SendThread(127.0.0.1:50390)) [n:127.0.0.1:57655_solr ] o.a.z.ClientCnxn Client session timed out, have not heard from server in 15393ms for sessionid 0x1608b03a8a30006
[junit4] 2> 640361 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1608b03a8a30006, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 640362 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=wrongHost&p=wrongHost&t=simple-dt&e=1514196762006&s=+C3mhPb70uJvAFQ2Mg7J1tuioAE="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:42 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:42 GMT
[junit4] 2> 640363 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyWrongHost-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyWrongHost
[junit4] 2> 640394 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testProxyValidateHost
[junit4] 2> 640396 INFO (qtp214175776-4007) [n:127.0.0.1:38227_solr ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2 and sendToOCPQueue=true
[junit4] 2> 640397 INFO (qtp214175776-4007) [n:127.0.0.1:38227_solr ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={doAs=bar&action=LIST&user.name=localHostAnyGroup&wt=javabin&version=2} status=0 QTime=0
[junit4] 2> 640397 WARN (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[BA3454F2C111D1B0]) [ ] o.a.h.c.p.ResponseProcessCookies Invalid cookie header: "Set-Cookie: hadoop.auth="u=localHostAnyGroup&p=localHostAnyGroup&t=simple-dt&e=1514196777077&s=nuWFqhd3a8bsLEcS/6+fxwKXrLE="; Domain=127.0.0.1; Expires=Th 2, 25-thg 12-2017 10:12:57 GMT; HttpOnly". Invalid 'expires' attribute: Th 2, 25-thg 12-2017 10:12:57 GMT
[junit4] 2> 640397 INFO (TEST-TestImpersonationWithHadoopAuth.testProxyValidateHost-seed#[BA3454F2C111D1B0]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testProxyValidateHost
[junit4] 2> 640461 INFO (jetty-launcher-665-thread-1-EventThread) [n:127.0.0.1:57655_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 640461 WARN (zkConnectionManagerCallback-682-thread-1-processing-n:127.0.0.1:38227_solr) [n:127.0.0.1:38227_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1d7f8f3d name: ZooKeeperConnection Watcher:127.0.0.1:50390/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 640461 WARN (zkConnectionManagerCallback-682-thread-1-processing-n:127.0.0.1:38227_solr) [n:127.0.0.1:38227_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 640465 WARN (zkConnectionManagerCallback-684-thread-1-processing-n:127.0.0.1:57655_solr) [n:127.0.0.1:57655_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@66a1a909 name: ZooKeeperConnection Watcher:127.0.0.1:50390/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 640465 WARN (zkConnectionManagerCallback-684-thread-1-processing-n:127.0.0.1:57655_solr) [n:127.0.0.1:57655_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 640468 INFO (jetty-closer-666-thread-2) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@4b8d3166{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 640469 INFO (jetty-closer-666-thread-1) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@3e5962dc{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 640470 INFO (jetty-closer-666-thread-2) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1451390537
[junit4] 2> 640470 INFO (jetty-closer-666-thread-1) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=534116522
[junit4] 2> 640470 INFO (jetty-closer-666-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 640470 INFO (jetty-closer-666-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@3355a747: rootName = solr_57655, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@35726c85
[junit4] 2> 640470 INFO (jetty-closer-666-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 640470 INFO (jetty-closer-666-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6c75b3: rootName = solr_38227, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@2b014a3b
[junit4] 2> 640479 INFO (jetty-closer-666-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 640480 INFO (jetty-closer-666-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2cfbcb29: rootName = solr_38227, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@56a127de
[junit4] 2> 640480 INFO (jetty-closer-666-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 640480 INFO (jetty-closer-666-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@4232f34b: rootName = solr_57655, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@56a127de
[junit4] 2> 640488 INFO (jetty-closer-666-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 640488 INFO (jetty-closer-666-thread-1) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@19aaa136: rootName = solr_38227, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@50acd0d6
[junit4] 2> 640489 INFO (jetty-closer-666-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 640489 INFO (jetty-closer-666-thread-2) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@18f915b8: rootName = solr_57655, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@50acd0d6
[junit4] 2> 641606 INFO (zkConnectionManagerCallback-684-thread-1-processing-n:127.0.0.1:57655_solr) [n:127.0.0.1:57655_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 641608 INFO (jetty-closer-666-thread-2) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:57655_solr
[junit4] 2> 641613 INFO (jetty-closer-666-thread-2) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 641613 INFO (jetty-closer-666-thread-2) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 641614 INFO (jetty-closer-666-thread-2) [ ] o.a.s.c.Overseer Overseer (id=99232039632568326-127.0.0.1:57655_solr-n_0000000000) closing
[junit4] 2> 641614 INFO (OverseerStateUpdate-99232039632568326-127.0.0.1:57655_solr-n_0000000000) [n:127.0.0.1:57655_solr ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57655_solr
[junit4] 2> 641614 INFO (OverseerCollectionConfigSetProcessor-99232039632568326-127.0.0.1:57655_solr-n_0000000000) [n:127.0.0.1:57655_solr ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=99232039632568326-127.0.0.1:57655_solr-n_0000000000) am no longer a leader.
[junit4] 2> 641614 INFO (zkCallback-683-thread-3-processing-n:127.0.0.1:57655_solr) [n:127.0.0.1:57655_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 641616 WARN (OverseerAutoScalingTriggerThread-99232039632568326-127.0.0.1:57655_solr-n_0000000000) [n:127.0.0.1:57655_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread woken up but we are closed, exiting.
[junit4] 2> 641618 WARN (ScheduledTrigger-1704-thread-1) [n:127.0.0.1:57655_solr ] o.a.s.c.a.TriggerBase Exception updating trigger state '/autoscaling/triggerState/.auto_add_replicas'
[junit4] 2> java.lang.InterruptedException: sleep interrupted
[junit4] 2> at java.lang.Thread.sleep(Native Method)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:111)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:315)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.hasData(ZkDistribStateManager.java:57)
[junit4] 2> at org.apache.solr.cloud.autoscaling.TriggerBase.saveState(TriggerBase.java:203)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$ScheduledTrigger.run(ScheduledTriggers.java:575)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
[junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
[junit4] 2> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 641625 ERROR (Thread[Thread-805,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:57655_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 641700 INFO (jetty-launcher-665-thread-2-EventThread) [n:127.0.0.1:38227_solr ] o.a.c.f.s.ConnectionStateManager State change: SUSPENDED
[junit4] 2> 641936 INFO (jetty-closer-666-thread-2) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@22ebd6a2{/solr,null,UNAVAILABLE}
[junit4] 2> 642248 INFO (zkConnectionManagerCallback-682-thread-1-processing-n:127.0.0.1:38227_solr) [n:127.0.0.1:38227_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 642250 INFO (zkCallback-681-thread-6-processing-n:127.0.0.1:38227_solr) [n:127.0.0.1:38227_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (1)
[junit4] 2> 642250 INFO (zkCallback-681-thread-7-processing-n:127.0.0.1:38227_solr) [n:127.0.0.1:38227_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:38227_solr
[junit4] 2> 642253 INFO (jetty-closer-666-thread-1) [ ] o.a.s.c.ZkController Remove node as live in ZooKeeper:/live_nodes/127.0.0.1:38227_solr
[junit4] 2> 642255 INFO (jetty-closer-666-thread-1) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.cluster, tag=null
[junit4] 2> 642255 INFO (jetty-closer-666-thread-1) [ ] o.a.s.c.TransientSolrCoreCacheDefault Allocating transient cache for 2147483647 transient cores
[junit4] 2> 642259 ERROR (Thread[Thread-803,5,TGRP-TestImpersonationWithHadoopAuth]) [n:127.0.0.1:38227_solr ] o.a.h.s.t.d.AbstractDelegationTokenSecretManager ExpiredTokenRemover received java.lang.InterruptedException: sleep interrupted
[junit4] 2> 642266 INFO (jetty-closer-666-thread-1) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@2454d891{/solr,null,UNAVAILABLE}
[junit4] 2> 642269 ERROR (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 642270 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50390 50390
[junit4] 2> 648936 INFO (Thread-786) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:50390 50390
[junit4] 2> 648937 WARN (Thread-786) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2> 3 /solr/clusterprops.json
[junit4] 2> 2 /solr/security.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMKeyIdRoot
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_1
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_3
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_2
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot/DK_4
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMSeqNumRoot
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMMasterKeyRoot
[junit4] 2> 2 /solr/security/zkdtsm/ZKDTSMRoot/ZKDTSMTokensRoot
[junit4] 2>
[junit4] 2> 648938 INFO (SUITE-TestImpersonationWithHadoopAuth-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 ------------------------------------------------------- Done waiting for tracked resources to be released
[junit4] 2> Dec 25, 2017 12:13:05 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
[junit4] 2> Dec 25, 2017 12:13:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
[junit4] 2> SEVERE: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
[junit4] 2> 1) Thread[id=4051, name=jetty-launcher-665-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 2) Thread[id=4055, name=jetty-launcher-665-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> at sun.misc.Unsafe.park(Native Method)
[junit4] 2> at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Dec 25, 2017 12:13:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: Starting to interrupt leaked threads:
[junit4] 2> 1) Thread[id=4051, name=jetty-launcher-665-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> 2) Thread[id=4055, name=jetty-launcher-665-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] 2> 668993 ERROR (jetty-launcher-665-thread-1-EventThread) [n:127.0.0.1:57655_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> 668993 ERROR (jetty-launcher-665-thread-2-EventThread) [n:127.0.0.1:38227_solr ] o.a.c.f.i.CuratorFrameworkImpl Watcher exception
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1039)
[junit4] 2> at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] 2> at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] 2> at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] 2> at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] 2> at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] 2> at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] 2> at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] 2> Dec 25, 2017 12:13:25 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
[junit4] 2> INFO: All leaked threads terminated.
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene70): {}, docValues:{}, maxPointsInLeafNode=1388, maxMBSortInHeap=5.985160730418684, sim=RandomSimilarity(queryNorm=false): {}, locale=vi, timezone=Europe/Mariehamn
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=105067736,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [TestSlowCompositeReaderWrapper, TestPullReplica, TestFaceting, EchoParamsTest, ClassificationUpdateProcessorTest, AddBlockUpdateTest, TestRandomFaceting, HdfsChaosMonkeySafeLeaderTest, ShardSplitTest, BufferStoreTest, DocValuesNotIndexedTest, TestComplexPhraseQParserPlugin, SolrIndexConfigTest, TestNodeLostTrigger, TestHttpShardHandlerFactory, RegexBytesRefFilterTest, TestBlobHandler, QueryResultKeyTest, TestTestInjection, CoreAdminHandlerTest, TestConfigSetsAPIExclusivity, BaseCdcrDistributedZkTest, PolyFieldTest, FileUtilsTest, TestLegacyNumericUtils, TestMaxScoreQueryParser, TestCloudInspectUtil, TestFieldSortValues, TestBadConfig, TolerantUpdateProcessorTest, HdfsAutoAddReplicasIntegrationTest, TestStressLucene, PreAnalyzedFieldTest, TestCollationFieldDocValues, TestScoreJoinQPScore, AnalysisAfterCoreReloadTest, TestImpersonationWithHadoopAuth]
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=TestImpersonationWithHadoopAuth -Dtests.seed=BA3454F2C111D1B0 -Dtests.slow=true -Dtests.locale=vi -Dtests.timezone=Europe/Mariehamn -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 0.00s J1 | TestImpersonationWithHadoopAuth (suite) <<<
[junit4] > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.security.hadoop.TestImpersonationWithHadoopAuth:
[junit4] > 1) Thread[id=4051, name=jetty-launcher-665-thread-1-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > 2) Thread[id=4055, name=jetty-launcher-665-thread-2-EventThread, state=TIMED_WAITING, group=TGRP-TestImpersonationWithHadoopAuth]
[junit4] > at sun.misc.Unsafe.park(Native Method)
[junit4] > at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037)
[junit4] > at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328)
[junit4] > at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277)
[junit4] > at org.apache.curator.CuratorZookeeperClient.internalBlockUntilConnectedOrTimedOut(CuratorZookeeperClient.java:323)
[junit4] > at org.apache.curator.RetryLoop.callWithRetry(RetryLoop.java:105)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.pathInForeground(GetDataBuilderImpl.java:288)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:279)
[junit4] > at org.apache.curator.framework.imps.GetDataBuilderImpl.forPath(GetDataBuilderImpl.java:41)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.readValue(SharedValue.java:244)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue.access$100(SharedValue.java:44)
[junit4] > at org.apache.curator.framework.recipes.shared.SharedValue$1.process(SharedValue.java:61)
[junit4] > at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:67)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:530)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:505)
[junit4] > at __randomizedtesting.SeedInfo.seed([BA3454F2C111D1B0]:0)
[junit4] Completed [101/764 (1!)] on J1 in 44.74s, 10 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 820 lines...]
[junit4] Suite: org.apache.solr.cloud.MoveReplicaHDFSTest
[junit4] 2> 2008199 INFO (SUITE-MoveReplicaHDFSTest-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> Creating dataDir: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BA3454F2C111D1B0-001/init-core-data-001
[junit4] 2> 2008199 WARN (SUITE-MoveReplicaHDFSTest-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=19 numCloses=19
[junit4] 2> 2008200 INFO (SUITE-MoveReplicaHDFSTest-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=false
[junit4] 2> 2008201 INFO (SUITE-MoveReplicaHDFSTest-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, value=NaN, ssl=NaN, clientAuth=NaN)
[junit4] 2> 2008202 INFO (SUITE-MoveReplicaHDFSTest-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 4 servers in /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J1/temp/solr.cloud.MoveReplicaHDFSTest_BA3454F2C111D1B0-001/tempDir-001
[junit4] 2> 2008202 INFO (SUITE-MoveReplicaHDFSTest-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 2008203 INFO (Thread-2695) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 2008203 INFO (Thread-2695) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 2008205 ERROR (Thread-2695) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 2008304 INFO (SUITE-MoveReplicaHDFSTest-seed#[BA3454F2C111D1B0]-worker) [ ] o.a.s.c.ZkTestServer start zk server on port:36483
[junit4] 2> 2008306 INFO (zkConnectionManagerCallback-4158-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008317 INFO (jetty-launcher-4155-thread-1) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2008317 INFO (jetty-launcher-4155-thread-3) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2008319 INFO (jetty-launcher-4155-thread-4) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2008320 INFO (jetty-launcher-4155-thread-3) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@24704386{/solr,null,AVAILABLE}
[junit4] 2> 2008320 INFO (jetty-launcher-4155-thread-2) [ ] o.e.j.s.Server jetty-9.3.20.v20170531
[junit4] 2> 2008320 INFO (jetty-launcher-4155-thread-3) [ ] o.e.j.s.AbstractConnector Started ServerConnector@16aaf4c3{HTTP/1.1,[http/1.1]}{127.0.0.1:33954}
[junit4] 2> 2008320 INFO (jetty-launcher-4155-thread-3) [ ] o.e.j.s.Server Started @2011919ms
[junit4] 2> 2008321 INFO (jetty-launcher-4155-thread-3) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=33954}
[junit4] 2> 2008321 ERROR (jetty-launcher-4155-thread-3) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2008321 INFO (jetty-launcher-4155-thread-2) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@6012ba70{/solr,null,AVAILABLE}
[junit4] 2> 2008321 INFO (jetty-launcher-4155-thread-2) [ ] o.e.j.s.AbstractConnector Started ServerConnector@373139f1{HTTP/1.1,[http/1.1]}{127.0.0.1:50103}
[junit4] 2> 2008321 INFO (jetty-launcher-4155-thread-2) [ ] o.e.j.s.Server Started @2011920ms
[junit4] 2> 2008321 INFO (jetty-launcher-4155-thread-2) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=50103}
[junit4] 2> 2008322 ERROR (jetty-launcher-4155-thread-2) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2008322 INFO (jetty-launcher-4155-thread-2) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2008322 INFO (jetty-launcher-4155-thread-1) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@79472bfc{/solr,null,AVAILABLE}
[junit4] 2> 2008322 INFO (jetty-launcher-4155-thread-3) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2008322 INFO (jetty-launcher-4155-thread-3) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2008322 INFO (jetty-launcher-4155-thread-3) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2008322 INFO (jetty-launcher-4155-thread-3) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T00:35:45.003Z
[junit4] 2> 2008323 INFO (jetty-launcher-4155-thread-4) [ ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@704d37d0{/solr,null,AVAILABLE}
[junit4] 2> 2008323 INFO (jetty-launcher-4155-thread-4) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4acc9c38{HTTP/1.1,[http/1.1]}{127.0.0.1:40294}
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-4) [ ] o.e.j.s.Server Started @2011922ms
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-1) [ ] o.e.j.s.AbstractConnector Started ServerConnector@4a94bde1{HTTP/1.1,[http/1.1]}{127.0.0.1:64692}
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-4) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=40294}
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-1) [ ] o.e.j.s.Server Started @2011922ms
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-1) [ ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=64692}
[junit4] 2> 2008324 ERROR (jetty-launcher-4155-thread-4) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2008324 ERROR (jetty-launcher-4155-thread-1) [ ] o.a.s.u.StartupLoggingUtils Missing Java Option solr.log.dir. Logging may be missing or incomplete.
[junit4] 2> 2008322 INFO (jetty-launcher-4155-thread-2) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-2) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-1) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-2) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T00:35:45.005Z
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-4) [ ] o.a.s.s.SolrDispatchFilter ___ _ Welcome to Apache Solr? version 7.3.0
[junit4] 2> 2008324 INFO (jetty-launcher-4155-thread-1) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2008325 INFO (jetty-launcher-4155-thread-4) [ ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _ Starting in cloud mode on port null
[junit4] 2> 2008325 INFO (jetty-launcher-4155-thread-1) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2008325 INFO (jetty-launcher-4155-thread-4) [ ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_| Install dir: null
[junit4] 2> 2008325 INFO (jetty-launcher-4155-thread-1) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T00:35:45.006Z
[junit4] 2> 2008325 INFO (jetty-launcher-4155-thread-4) [ ] o.a.s.s.SolrDispatchFilter |___/\___/_|_| Start time: 2017-12-25T00:35:45.006Z
[junit4] 2> 2008367 INFO (zkConnectionManagerCallback-4160-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008368 INFO (zkConnectionManagerCallback-4162-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008369 INFO (jetty-launcher-4155-thread-2) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2008369 INFO (zkConnectionManagerCallback-4164-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008369 INFO (zkConnectionManagerCallback-4166-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008371 INFO (jetty-launcher-4155-thread-4) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2008372 INFO (jetty-launcher-4155-thread-3) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2008373 INFO (jetty-launcher-4155-thread-1) [ ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
[junit4] 2> 2008709 INFO (jetty-launcher-4155-thread-2) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36483/solr
[junit4] 2> 2008711 INFO (jetty-launcher-4155-thread-1) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36483/solr
[junit4] 2> 2008714 INFO (jetty-launcher-4155-thread-4) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36483/solr
[junit4] 2> 2008715 INFO (jetty-launcher-4155-thread-3) [ ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:36483/solr
[junit4] 2> 2008711 INFO (zkConnectionManagerCallback-4170-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008717 INFO (zkConnectionManagerCallback-4178-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008718 INFO (zkConnectionManagerCallback-4180-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008719 INFO (zkConnectionManagerCallback-4182-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008719 INFO (zkConnectionManagerCallback-4184-thread-1-processing-n:127.0.0.1:50103_solr) [n:127.0.0.1:50103_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008721 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x1608b18c6750007, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:239)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 2008722 INFO (zkConnectionManagerCallback-4186-thread-1-processing-n:127.0.0.1:64692_solr) [n:127.0.0.1:64692_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008723 INFO (zkConnectionManagerCallback-4188-thread-1-processing-n:127.0.0.1:40294_solr) [n:127.0.0.1:40294_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008723 INFO (zkConnectionManagerCallback-4190-thread-1-processing-n:127.0.0.1:33954_solr) [n:127.0.0.1:33954_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 2008768 INFO (jetty-launcher-4155-thread-3) [n:127.0.0.1:33954_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 2008
[...truncated too long message...]
4F2C111D1B0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2435698 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@b1bd1c: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@25da53ee
[junit4] 2> 2435698 INFO (coreCloseExecutor-8469-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@1a8ca707
[junit4] 2> 2435698 INFO (coreCloseExecutor-8469-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=445425415
[junit4] 2> 2435698 INFO (coreCloseExecutor-8469-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@35b0aec6: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@5c06eb1b
[junit4] 2> 2435710 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@37ff1ec7{/solr,null,UNAVAILABLE}
[junit4] 2> 2435713 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.e.j.s.AbstractConnector Stopped ServerConnector@3ded54cd{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
[junit4] 2> 2435714 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1155788195
[junit4] 2> 2435717 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 2435718 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@6af65bd5: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@229e5b9b
[junit4] 2> 2435724 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jvm, tag=null
[junit4] 2> 2435725 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@334a0e82: rootName = null, domain = solr.jvm, service url = null, agent id = null] for registry solr.jvm / com.codahale.metrics.MetricRegistry@2de0bebc
[junit4] 2> 2435731 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.jetty, tag=null
[junit4] 2> 2435731 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@7ad294f6: rootName = null, domain = solr.jetty, service url = null, agent id = null] for registry solr.jetty / com.codahale.metrics.MetricRegistry@25da53ee
[junit4] 2> 2435731 INFO (coreCloseExecutor-8471-thread-1) [ x:collection1] o.a.s.c.SolrCore [collection1] CLOSING SolrCore org.apache.solr.core.SolrCore@1a938220
[junit4] 2> 2435731 INFO (coreCloseExecutor-8471-thread-1) [ x:collection1] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.core.collection1, tag=445874720
[junit4] 2> 2435732 INFO (coreCloseExecutor-8471-thread-1) [ x:collection1] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@40252f8: rootName = null, domain = solr.core.collection1, service url = null, agent id = null] for registry solr.core.collection1 / com.codahale.metrics.MetricRegistry@1ea2c83b
[junit4] 2> 2435747 INFO (TEST-TestReplicationHandler.doTestReplicateAfterCoreReload-seed#[BA3454F2C111D1B0]) [ ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@6358d21c{/solr,null,UNAVAILABLE}
[junit4] IGNOR/A 0.00s J0 | TestReplicationHandler.doTestIndexFetchOnMasterRestart
[junit4] > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/SOLR-9036))
[junit4] 2> NOTE: leaving temporary files on disk at: /export/home/jenkins/workspace/Lucene-Solr-7.x-Solaris/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_BA3454F2C111D1B0-001
[junit4] 2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarity(queryNorm=false): {}, locale=it-CH, timezone=America/Nome
[junit4] 2> NOTE: SunOS 5.11 amd64/Oracle Corporation 1.8.0_152 (64-bit)/cpus=3,threads=1,free=298168352,total=536870912
[junit4] 2> NOTE: All tests run in this JVM: [UninvertDocValuesMergePolicyTest, TestMiniSolrCloudClusterSSL, TestIntervalFaceting, MinimalSchemaTest, CloneFieldUpdateProcessorFactoryTest, AnalysisErrorHandlingTest, TestJoin, HdfsBasicDistributedZk2Test, ShardRoutingTest, HdfsDirectoryTest, ForceLeaderTest, ReturnFieldsTest, TestAtomicUpdateErrorCases, TestGenericDistributedQueue, FastVectorHighlighterTest, SolrSlf4jReporterTest, FieldAnalysisRequestHandlerTest, HdfsDirectoryFactoryTest, UtilsToolTest, V2StandaloneTest, TestCloudSearcherWarming, TestPolicyCloud, NotRequiredUniqueKeyTest, CircularListTest, TestPayloadCheckQParserPlugin, SolrXmlInZkTest, TestFiltering, TestBackupRepositoryFactory, TimeRoutedAliasUpdateProcessorTest, AutoScalingHandlerTest, SortByFunctionTest, PrimitiveFieldTypeTest, TestHalfAndHalfDocValues, TestPerFieldSimilarityWithDefaultOverride, ConvertedLegacyTest, ConjunctionSolrSpellCheckerTest, BlockJoinFacetSimpleTest, SolrGangliaReporterTest, TestMacroExpander, TestNodeAddedTrigger, RecoveryZkTest, TestTolerantSearch, TestDocTermOrds, TestSolrCoreProperties, CSVRequestHandlerTest, HLLUtilTest, TestIBSimilarityFactory, MoveReplicaHDFSFailoverTest, TestMissingGroups, TestJsonFacetRefinement, DefaultValueUpdateProcessorTest, TestImplicitCoreProperties, ShowFileRequestHandlerTest, LeaderElectionTest, TestRandomRequestDistribution, UUIDFieldTest, PreAnalyzedUpdateProcessorTest, TestPointFields, JvmMetricsTest, TestTrieFacet, CollectionReloadTest, TestSurroundQueryParser, TestCloudSchemaless, TestNumericTokenStream, TestObjectReleaseTracker, TestSubQueryTransformer, TestLegacyTerms, AddReplicaTest, HighlighterConfigTest, TestCloudNestedDocsSort, PeerSyncReplicationTest, NumberUtilsTest, DirectUpdateHandlerTest, MoveReplicaTest, TestBinaryField, TestNumericRangeQuery32, SuggesterWFSTTest, BigEndianAscendingWordDeserializerTest, XsltUpdateRequestHandlerTest, TestFuzzyAnalyzedSuggestions, UUIDUpdateProcessorFallbackTest, DirectSolrConnectionTest, BlockJoinFacetDistribTest, TestReversedWildcardFilterFactory, TestSimpleTextCodec, UpdateLogTest, TestConfigSets, TestSolrConfigHandlerCloud, TestSQLHandler, DirectoryFactoryTest, DistributedFacetPivotSmallAdvancedTest, XmlUpdateRequestHandlerTest, TestPolicyCloud, TestUseDocValuesAsStored2, TestLazyCores, BlockDirectoryTest, TestPhraseSuggestions, BadIndexSchemaTest, SolrMetricReporterTest, TestManagedStopFilterFactory, TestHighlightDedupGrouping, RequestHandlersTest, ChaosMonkeyNothingIsSafeTest, TestSystemCollAutoCreate, HighlighterTest, VMParamsZkACLAndCredentialsProvidersTest, ConnectionReuseTest, TestReloadAndDeleteDocs, DistributedQueryElevationComponentTest, TestSchemalessBufferedUpdates, UnloadDistributedZkTest, MetricUtilsTest, TestOmitPositions, TestTolerantUpdateProcessorRandomCloud, TestReplicaProperties, TestHashQParserPlugin, SolrIndexMetricsTest, SuggesterTSTTest, HttpTriggerListenerTest, PrimUtilsTest, TestConfigSetImmutable, TestAuthorizationFramework, TestNumericTerms32, TestExpandComponent, CollectionsAPISolrJTest, BasicDistributedZkTest, TestBinaryResponseWriter, TestSerializedLuceneMatchVersion, TestPHPSerializedResponseWriter, TestSolrCloudWithSecureImpersonation, JavabinLoaderTest, NumericFieldsTest, ExecutePlanActionTest, TestManagedResourceStorage, TestQuerySenderListener, TlogReplayBufferedWhileIndexingTest, LargeFieldTest, SpatialHeatmapFacetsTest, RAMDirectoryFactoryTest, StatsReloadRaceTest, TestScoreJoinQPNoScore, TestUtils, DeleteInactiveReplicaTest, TestDistribStateManager, OverriddenZkACLAndCredentialsProvidersTest, TestExceedMaxTermLength, TestInitParams, TestExportWriter, TestInPlaceUpdatesDistrib, TestSolrCloudWithHadoopAuthPlugin, TestDelegationWithHadoopAuth, TestTriggerIntegration, CollectionTooManyReplicasTest, TestAnalyzeInfixSuggestions, NodeMutatorTest, IndexSchemaTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, FacetPivotSmallTest, TestCoreDiscovery, TestFieldCacheSortRandom, OverseerModifyCollectionTest, TestStressInPlaceUpdates, TestSubQueryTransformerDistrib, LeaderElectionContextKeyTest, DistribJoinFromCollectionTest, SuggesterTest, CurrencyFieldTypeTest, RegexBoostProcessorTest, JSONWriterTest, TestDFISimilarityFactory, TestSearcherReuse, HdfsRecoverLeaseTest, SimpleMLTQParserTest, TestRealTimeGet, TestSolrCloudWithDelegationTokens, TestCodecSupport, ClassificationUpdateProcessorIntegrationTest, ZkControllerTest, URLClassifyProcessorTest, AutoAddReplicasIntegrationTest, TestRecovery, ReplicaListTransformerTest, NoCacheHeaderTest, TestDFRSimilarityFactory, TestCloudJSONFacetJoinDomain, ChaosMonkeySafeLeaderTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, TestReplicationHandler]
[junit4] Completed [390/764 (3!)] on J0 in 88.69s, 16 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 49419 lines...]