You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/04/02 22:06:55 UTC
[JENKINS] Lucene-Solr-NightlyTests-7.3 - Build # 17 - Still
Unstable
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-7.3/17/
3 tests failed.
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry
Error Message:
ObjectTracker found 1 object(s) that were not released!!! [Overseer] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42) at org.apache.solr.cloud.Overseer.start(Overseer.java:545) at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850) at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170) at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307) at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216) at org.apache.solr.cloud.ZkController$1.command(ZkController.java:355) at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167) at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57) at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141) at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
Stack Trace:
java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
at org.apache.solr.cloud.ZkController$1.command(ZkController.java:355)
at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
at __randomizedtesting.SeedInfo.seed([3D69AF76A3E5BD47]:0)
at org.junit.Assert.fail(Assert.java:93)
at org.junit.Assert.assertTrue(Assert.java:43)
at org.junit.Assert.assertNull(Assert.java:551)
at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:301)
at sun.reflect.GeneratedMethodAccessor55.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:897)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:368)
at java.lang.Thread.run(Thread.java:748)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry
Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry: 1) Thread[id=17063, name=OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at java.lang.Throwable.fillInStackTrace(Native Method) at java.lang.Throwable.fillInStackTrace(Throwable.java:783) at java.lang.Throwable.<init>(Throwable.java:250) at java.lang.Exception.<init>(Exception.java:54) at org.apache.zookeeper.KeeperException.<init>(KeeperException.java:439) at org.apache.zookeeper.KeeperException$SessionExpiredException.<init>(KeeperException.java:685) at org.apache.zookeeper.KeeperException.create(KeeperException.java:130) at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215) at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340) at org.apache.solr.common.cloud.SolrZkClient$$Lambda$217/191921242.execute(Unknown Source) at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60) at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340) at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176) at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry:
1) Thread[id=17063, name=OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
at java.lang.Throwable.fillInStackTrace(Native Method)
at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
at java.lang.Throwable.<init>(Throwable.java:250)
at java.lang.Exception.<init>(Exception.java:54)
at org.apache.zookeeper.KeeperException.<init>(KeeperException.java:439)
at org.apache.zookeeper.KeeperException$SessionExpiredException.<init>(KeeperException.java:685)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
at org.apache.solr.common.cloud.SolrZkClient$$Lambda$217/191921242.execute(Unknown Source)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([3D69AF76A3E5BD47]:0)
FAILED: junit.framework.TestSuite.org.apache.solr.cloud.TestLeaderElectionZkExpiry
Error Message:
There are still zombie threads that couldn't be terminated: 1) Thread[id=17063, name=OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers] at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128) at java.io.PrintStream.write(PrintStream.java:480) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576) at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:140) at java.lang.Thread.run(Thread.java:748)
Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
1) Thread[id=17063, name=OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
at org.apache.lucene.util.TestRuleLimitSysouts$DelegateStream.write(TestRuleLimitSysouts.java:128)
at java.io.PrintStream.write(PrintStream.java:480)
at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
at org.apache.log4j.Category.callAppenders(Category.java:206)
at org.apache.log4j.Category.forcedLog(Category.java:391)
at org.apache.log4j.Category.log(Category.java:856)
at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:140)
at java.lang.Thread.run(Thread.java:748)
at __randomizedtesting.SeedInfo.seed([3D69AF76A3E5BD47]:0)
Build Log:
[...truncated 13877 lines...]
[junit4] Suite: org.apache.solr.cloud.TestLeaderElectionZkExpiry
[junit4] 2> Creating dataDir: /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLeaderElectionZkExpiry_3D69AF76A3E5BD47-001/init-core-data-001
[junit4] 2> 3049360 WARN (SUITE-TestLeaderElectionZkExpiry-seed#[3D69AF76A3E5BD47]-worker) [ ] o.a.s.SolrTestCaseJ4 startTrackingSearchers: numOpens=35 numCloses=35
[junit4] 2> 3049360 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[3D69AF76A3E5BD47]-worker) [ ] o.a.s.SolrTestCaseJ4 Using PointFields (NUMERIC_POINTS_SYSPROP=true) w/NUMERIC_DOCVALUES_SYSPROP=true
[junit4] 2> 3049362 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[3D69AF76A3E5BD47]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
[junit4] 2> 3049375 INFO (SUITE-TestLeaderElectionZkExpiry-seed#[3D69AF76A3E5BD47]-worker) [ ] o.a.s.SolrTestCaseJ4 SecureRandom sanity checks: test.solr.allowed.securerandom=null & java.security.egd=file:/dev/./urandom
[junit4] 2> 3049413 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testLeaderElectionWithZkExpiry
[junit4] 2> 3049418 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.c.SolrXmlConfig MBean server found: com.sun.jmx.mbeanserver.JmxMBeanServer@1b0ef58c, but no JMX reporters were configured - adding default JMX reporter.
[junit4] 2> 3049584 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.node' (registry 'solr.node') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b0ef58c
[junit4] 2> 3049612 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jvm' (registry 'solr.jvm') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b0ef58c
[junit4] 2> 3049612 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.m.r.SolrJmxReporter JMX monitoring for 'solr.jetty' (registry 'solr.jetty') enabled at server: com.sun.jmx.mbeanserver.JmxMBeanServer@1b0ef58c
[junit4] 2> 3049613 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-7.3/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.TestLeaderElectionZkExpiry_3D69AF76A3E5BD47-001/testLeaderElectionWithZkExpiry-solr-001
[junit4] 2> 3049631 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 3049631 INFO (Thread-4536) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 3049632 INFO (Thread-4536) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 3049683 ERROR (Thread-4536) [ ] o.a.z.s.ZooKeeperServer ZKShutdownHandler is not registered, so ZooKeeper server won't take any action on ERROR or SHUTDOWN server state changes
[junit4] 2> 3049731 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [ ] o.a.s.c.ZkTestServer start zk server on port:32935
[junit4] 2> 3049734 INFO (zkConnectionManagerCallback-3946-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3049736 INFO (zkConnectionManagerCallback-3948-thread-1) [ ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3049739 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3049947 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 3049948 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader dummy.host.com:8984_solr
[junit4] 2> 3049949 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=72387939146268674-dummy.host.com:8984_solr-n_0000000000) starting
[junit4] 2> 3049963 INFO (zkConnectionManagerCallback-3955-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3049965 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.c.s.i.ZkClientClusterStateProvider Cluster at 127.0.0.1:32935/solr ready
[junit4] 2> 3049978 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/dummy.host.com:8984_solr
[junit4] 2> 3050001 INFO (zkCallback-3949-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3050028 INFO (zkCallback-3954-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 3050030 INFO (zkCallback-3954-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 3050130 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3050131 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3052120 WARN (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]-SendThread(127.0.0.1:32935)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1012c7276d60002 has expired
[junit4] 2> 3052120 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 3052120 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 3052120 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=72387939146268674-dummy.host.com:8984_solr-n_0000000000) closing
[junit4] 2> 3052120 INFO (OverseerStateUpdate-72387939146268674-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer Loop exiting : dummy.host.com:8984_solr
[junit4] 2> 3052121 INFO (OverseerCollectionConfigSetProcessor-72387939146268674-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=72387939146268674-dummy.host.com:8984_solr-n_0000000000) am no longer a leader.
[junit4] 2> 3052122 ERROR (OverseerExitThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:306)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3052220 WARN (zkCallback-3949-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 3052220 ERROR (OverseerAutoScalingTriggerThread-72387939146268674-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr ] o.a.s.c.a.ScheduledTriggers Failed to add trigger .auto_add_replicas
[junit4] 2> org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
[junit4] 2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:124)
[junit4] 2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:114)
[junit4] 2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:110)
[junit4] 2> at org.apache.solr.cloud.ZkDistributedQueueFactory.makeQueue(ZkDistributedQueueFactory.java:36)
[junit4] 2> at org.apache.solr.cloud.autoscaling.TriggerEventQueue.<init>(TriggerEventQueue.java:49)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers$TriggerWrapper.<init>(ScheduledTriggers.java:511)
[junit4] 2> at org.apache.solr.cloud.autoscaling.ScheduledTriggers.add(ScheduledTriggers.java:204)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:227)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling/events/.auto_add_replicas
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$2(SolrZkClient.java:304)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:304)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:519)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:97)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:78)
[junit4] 2> at org.apache.solr.cloud.ZkDistributedQueue.<init>(ZkDistributedQueue.java:122)
[junit4] 2> ... 8 more
[junit4] 2> 3052221 WARN (OverseerAutoScalingTriggerThread-72387939146268674-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread Error removing old nodeAdded markers
[junit4] 2> java.lang.InterruptedException
[junit4] 2> at java.lang.Object.wait(Native Method)
[junit4] 2> at java.lang.Object.wait(Object.java:502)
[junit4] 2> at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1409)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1533)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getChildren$4(SolrZkClient.java:328)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.listData(ZkDistribStateManager.java:67)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.listData(ZkDistribStateManager.java:77)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:254)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3052221 WARN (OverseerAutoScalingTriggerThread-72387939146268674-dummy.host.com:8984_solr-n_0000000000) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread OverseerTriggerThread has been closed, exiting.
[junit4] 2> 3052221 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 3052222 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3052222 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 3052223 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 3052224 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 3052325 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3052326 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3054169 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxnFactory Ignoring unexpected runtime exception
[junit4] 2> java.nio.channels.CancelledKeyException
[junit4] 2> at sun.nio.ch.SelectionKeyImpl.ensureValid(SelectionKeyImpl.java:73)
[junit4] 2> at sun.nio.ch.SelectionKeyImpl.readyOps(SelectionKeyImpl.java:87)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:205)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3054169 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:32935)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1012c7276d60004 has expired
[junit4] 2> 3054170 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 3054170 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 3054170 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 3054185 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3054185 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 3054185 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 3054275 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /aliases.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$2(SolrZkClient.java:304)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:304)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.refreshAliases(ZkStateReader.java:1589)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:441)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:267)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 3054275 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:415)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$1.lambda$process$0(SolrZkClient.java:267)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:188)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /aliases.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$2(SolrZkClient.java:304)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:304)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.refreshAliases(ZkStateReader.java:1589)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:441)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> ... 10 more
[junit4] 2>
[junit4] 2> 3054276 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
[junit4] 2> 3054276 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 3054303 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3054303 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3055929 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:32935)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1012c7276d60005 has expired
[junit4] 2> 3055929 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 3055929 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 3055929 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 3055930 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3055930 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 3055930 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 3056029 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2>
[junit4] 2> 3056030 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:415)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> ... 5 more
[junit4] 2>
[junit4] 2> 3056030 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
[junit4] 2> 3056030 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 3056032 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3056032 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3057976 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:32935)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1012c7276d60006 has expired
[junit4] 2> 3057976 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 3057977 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 3057977 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 3057986 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3057986 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 3057986 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 3058077 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2>
[junit4] 2> 3058077 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:415)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> ... 5 more
[junit4] 2>
[junit4] 2> 3058077 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
[junit4] 2> 3058078 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 3058090 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3058091 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3059619 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:32935)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1012c7276d60007 has expired
[junit4] 2> 3059620 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 3059620 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 3059620 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 3059621 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3059621 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 3059621 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 3059720 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2>
[junit4] 2> 3059720 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:415)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> ... 5 more
[junit4] 2>
[junit4] 2> 3059721 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
[junit4] 2> 3059721 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 3059725 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Disconnected type:None path:null path: null type: None
[junit4] 2> 3059725 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has disconnected
[junit4] 2> 3060040 INFO (zkConnectionManagerCallback-3957-thread-1-processing-n:dummy.host.com:8984_solr) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3061517 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-SendThread(127.0.0.1:32935)) [n:dummy.host.com:8984_solr ] o.a.z.ClientCnxn Unable to reconnect to ZooKeeper service, session 0x1012c7276d60008 has expired
[junit4] 2> 3061517 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@147d0e16 name: ZooKeeperConnection Watcher:127.0.0.1:32935/solr got event WatchedEvent state:Expired type:None path:null path: null type: None
[junit4] 2> 3061517 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Our previous ZooKeeper session was expired. Attempting to reconnect to recover relationship with ZooKeeper...
[junit4] 2> 3061517 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Connection expired - starting a new one...
[junit4] 2> 3061518 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient has connected
[junit4] 2> 3061518 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager Connection with ZooKeeper reestablished.
[junit4] 2> 3061518 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController ZooKeeper session re-connected ... refreshing core states after session expiration.
[junit4] 2> 3061619 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2>
[junit4] 2> 3061619 ERROR (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed:org.apache.solr.common.cloud.ZooKeeperException:
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:415)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] 2> at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] 2> at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1105)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$exists$3(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:316)
[junit4] 2> at org.apache.solr.common.cloud.ZkStateReader.createClusterStateWatchersAndUpdate(ZkStateReader.java:430)
[junit4] 2> at org.apache.solr.cloud.ZkController$1.command(ZkController.java:330)
[junit4] 2> ... 5 more
[junit4] 2>
[junit4] 2> 3061619 WARN (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnect to ZooKeeper failed
[junit4] 2> 3061620 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 3061642 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerElectionContext I am going to be the leader dummy.host.com:8984_solr
[junit4] 2> 3061656 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.c.TestLeaderElectionZkExpiry Time=5040678350061508 Overseer leader is = dummy.host.com:8984_solr
[junit4] 2> 3061656 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer (id=72387939146268682-dummy.host.com:8984_solr-n_0000000001) starting
[junit4] 2> 3061668 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/dummy.host.com:8984_solr
[junit4] 2> 3061669 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.DefaultConnectionStrategy Reconnected to ZooKeeper
[junit4] 2> 3061669 INFO (zkConnectionManagerCallback-3950-thread-1-processing-n:dummy.host.com:8984_solr-EventThread) [n:dummy.host.com:8984_solr ] o.a.s.c.c.ConnectionManager zkClient Connected:true
[junit4] 2> 3061670 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn Unable to read additional data from client sessionid 0x1012c7276d6000a, likely client has closed socket
[junit4] 2> 3061684 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061684 INFO (OverseerCollectionConfigSetProcessor-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=72387939146268682-dummy.host.com:8984_solr-n_0000000001) am no longer a leader.
[junit4] 2> 3061685 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061684 INFO (OverseerStateUpdate-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer According to ZK I (id=72387939146268682-dummy.host.com:8984_solr-n_0000000001) am no longer a leader.
[junit4] 2> 3061685 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061685 INFO (OverseerStateUpdate-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer According to ZK I (id=72387939146268682-dummy.host.com:8984_solr-n_0000000001) am no longer a leader.
[junit4] 2> 3061685 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061685 ERROR (OverseerCollectionConfigSetProcessor-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerTaskProcessor :org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/collection-queue-work
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1535)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getChildren$4(SolrZkClient.java:328)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:328)
[junit4] 2> at org.apache.solr.cloud.ZkDistributedQueue.fetchZkChildren(ZkDistributedQueue.java:410)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskQueue.getTailId(OverseerTaskQueue.java:272)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:163)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2>
[junit4] 2> 3061686 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061685 INFO (OverseerStateUpdate-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer Overseer Loop exiting : dummy.host.com:8984_solr
[junit4] 2> 3061686 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061686 INFO (OverseerCollectionConfigSetProcessor-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=72387939146268682-dummy.host.com:8984_solr-n_0000000001) am no longer a leader.
[junit4] 2> 3061686 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061686 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061713 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061713 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061687 ERROR (OverseerExitThread) [n:dummy.host.com:8984_solr ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:306)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> 3061714 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061713 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=285075914
[junit4] 2> 3061714 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061714 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.m.SolrMetricManager Closing metric reporters for registry=solr.node, tag=null
[junit4] 2> 3061714 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3061714 INFO (TEST-TestLeaderElectionZkExpiry.testLeaderElectionWithZkExpiry-seed#[3D69AF76A3E5BD47]) [n:dummy.host.com:8984_solr ] o.a.s.m.r.SolrJmxReporter Closing reporter [org.apache.solr.metrics.reporters.SolrJmxReporter@2cac5f79: rootName = null, domain = solr.node, service url = null, agent id = null] for registry solr.node / com.codahale.metrics.MetricRegistry@45ef88a
[junit4] 2> 3061715 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManag
[...truncated too long message...]
org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3166461 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] 2> 3166461 ERROR (OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001) [n:dummy.host.com:8984_solr ] o.a.s.c.a.OverseerTriggerThread A ZK error has occurred
[junit4] 2> java.io.IOException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:183)
[junit4] 2> at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] 2> at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] 2> at java.lang.Thread.run(Thread.java:748)
[junit4] 2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /autoscaling.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] 2> at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] 2> ... 3 more
[junit4] ERROR 0.00s J2 | TestLeaderElectionZkExpiry (suite) <<<
[junit4] > Throwable #1: java.lang.AssertionError: ObjectTracker found 1 object(s) that were not released!!! [Overseer]
[junit4] > org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException: org.apache.solr.cloud.Overseer
[junit4] > at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:42)
[junit4] > at org.apache.solr.cloud.Overseer.start(Overseer.java:545)
[junit4] > at org.apache.solr.cloud.OverseerElectionContext.runLeaderProcess(ElectionContext.java:850)
[junit4] > at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:170)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:135)
[junit4] > at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:307)
[junit4] > at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:216)
[junit4] > at org.apache.solr.cloud.ZkController$1.command(ZkController.java:355)
[junit4] > at org.apache.solr.common.cloud.ConnectionManager$1.update(ConnectionManager.java:167)
[junit4] > at org.apache.solr.common.cloud.DefaultConnectionStrategy.reconnect(DefaultConnectionStrategy.java:57)
[junit4] > at org.apache.solr.common.cloud.ConnectionManager.process(ConnectionManager.java:141)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:531)
[junit4] > at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:506)
[junit4] > at __randomizedtesting.SeedInfo.seed([3D69AF76A3E5BD47]:0)
[junit4] > at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:301)
[junit4] > at java.lang.Thread.run(Thread.java:748)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.TestLeaderElectionZkExpiry:
[junit4] > 1) Thread[id=17063, name=OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
[junit4] > at java.lang.Throwable.fillInStackTrace(Native Method)
[junit4] > at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
[junit4] > at java.lang.Throwable.<init>(Throwable.java:250)
[junit4] > at java.lang.Exception.<init>(Exception.java:54)
[junit4] > at org.apache.zookeeper.KeeperException.<init>(KeeperException.java:439)
[junit4] > at org.apache.zookeeper.KeeperException$SessionExpiredException.<init>(KeeperException.java:685)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:130)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:54)
[junit4] > at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1215)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.lambda$getData$5(SolrZkClient.java:340)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient$$Lambda$217/191921242.execute(Unknown Source)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:340)
[junit4] > at org.apache.solr.client.solrj.impl.ZkDistribStateManager.getAutoScalingConfig(ZkDistribStateManager.java:176)
[junit4] > at org.apache.solr.client.solrj.cloud.autoscaling.DistribStateManager.getAutoScalingConfig(DistribStateManager.java:78)
[junit4] > at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:126)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([3D69AF76A3E5BD47]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
[junit4] > 1) Thread[id=17063, name=OverseerAutoScalingTriggerThread-72387939146268682-dummy.host.com:8984_solr-n_0000000001, state=RUNNABLE, group=Overseer autoscaling triggers]
[junit4] > at java.io.PrintStream.write(PrintStream.java:480)
[junit4] > at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221)
[junit4] > at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291)
[junit4] > at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295)
[junit4] > at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141)
[junit4] > at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229)
[junit4] > at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59)
[junit4] > at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324)
[junit4] > at org.apache.log4j.WriterAppender.append(WriterAppender.java:162)
[junit4] > at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251)
[junit4] > at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66)
[junit4] > at org.apache.log4j.Category.callAppenders(Category.java:206)
[junit4] > at org.apache.log4j.Category.forcedLog(Category.java:391)
[junit4] > at org.apache.log4j.Category.log(Category.java:856)
[junit4] > at org.slf4j.impl.Log4jLoggerAdapter.error(Log4jLoggerAdapter.java:576)
[junit4] > at org.apache.solr.cloud.autoscaling.OverseerTriggerThread.run(OverseerTriggerThread.java:140)
[junit4] > at java.lang.Thread.run(Thread.java:748)
[junit4] > at __randomizedtesting.SeedInfo.seed([3D69AF76A3E5BD47]:0)
[junit4] Completed [266/783 (1!)] on J2 in 117.26s, 1 test, 1 failure, 2 errors <<< FAILURES!
[...truncated 51147 lines...]