You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2016/02/21 07:57:09 UTC
[JENKINS] Lucene-Solr-Tests-trunk-Java8 - Build # 847 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-Tests-trunk-Java8/847/
1 tests failed.
FAILED: org.apache.solr.cloud.OverseerTest.testShardLeaderChange
Error Message:
Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
Stack Trace:
org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
at __randomizedtesting.SeedInfo.seed([58ADC6D3DE86652B:86FE4124C41E90DA]:0)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:212)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:173)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:138)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:310)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:219)
at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:181)
at org.apache.solr.cloud.OverseerTest.testShardLeaderChange(OverseerTest.java:841)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists
at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
at org.apache.zookeeper.ZooKeeper.multiInternal(ZooKeeper.java:949)
at org.apache.zookeeper.ZooKeeper.multi(ZooKeeper.java:915)
at org.apache.solr.common.cloud.SolrZkClient$11.execute(SolrZkClient.java:577)
at org.apache.solr.common.cloud.SolrZkClient$11.execute(SolrZkClient.java:574)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.multi(SolrZkClient.java:574)
at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:195)
at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:49)
at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:42)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:178)
... 45 more
Build Log:
[...truncated 11341 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/build/solr-core/test/J1/temp/solr.cloud.OverseerTest_58ADC6D3DE86652B-001/init-core-data-001
[junit4] 2> 1092525 INFO (SUITE-OverseerTest-seed#[58ADC6D3DE86652B]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false)
[junit4] 2> 1092525 INFO (SUITE-OverseerTest-seed#[58ADC6D3DE86652B]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 1092525 INFO (SUITE-OverseerTest-seed#[58ADC6D3DE86652B]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 1092536 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOverseerStatsReset
[junit4] 2> 1092536 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1092538 INFO (Thread-6683) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1092538 INFO (Thread-6683) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1092638 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer start zk server on port:56027
[junit4] 2> 1092638 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1092641 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1092644 INFO (zkCallback-2474-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@52567d01 name:ZooKeeperConnection Watcher:127.0.0.1:56027 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1092644 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1092645 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1092657 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1092659 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1092664 INFO (zkCallback-2475-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2280b03b name:ZooKeeperConnection Watcher:127.0.0.1:56027 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1092664 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1092664 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1092664 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 1092668 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1092679 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1092691 INFO (zkCallback-2476-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@56fc83ca name:ZooKeeperConnection Watcher:127.0.0.1:56027/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1092691 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1092691 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1092693 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 1092697 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 1092698 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 1092700 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 1092703 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 1092705 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1092706 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1092709 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1092710 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1092727 INFO (zkCallback-2477-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2f8021c3 name:ZooKeeperConnection Watcher:127.0.0.1:56027/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1092728 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1092728 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1092730 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1092732 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1092733 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1
[junit4] 2> 1092735 INFO (zkCallback-2477-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1092737 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 1092737 INFO (zkCallback-2477-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1092737 INFO (zkCallback-2476-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1092738 INFO (zkCallback-2476-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1092741 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 1092746 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 1092749 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1092750 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95422824626585602-127.0.0.1:56027_solr-n_0000000000
[junit4] 2> 1092750 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56027_solr
[junit4] 2> 1092750 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 1092751 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422824626585602-127.0.0.1:56027_solr-n_0000000000) starting
[junit4] 2> 1092752 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 1092756 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 1092761 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 1092764 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 1092766 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 1092783 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 1092790 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1092794 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 1092796 INFO (OverseerCollectionConfigSetProcessor-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 1092796 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
[junit4] 2> 1092800 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"} current state version: 0
[junit4] 2> 1092801 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1092801 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 1092801 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 1092802 INFO (zkCallback-2477-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1092804 INFO (zkCallback-2476-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1093300 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1093301 INFO (zkCallback-2477-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 1093301 INFO (zkCallback-2476-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 1093306 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95422824626585603-node1_core1-n_0000000000
[junit4] 2> 1093306 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1093309 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard1/leader after winning as /collections/collection1/leader_elect/shard1/election/95422824626585603-node1_core1-n_0000000000
[junit4] 2> 1093312 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422824626585602-127.0.0.1:56027_solr-n_0000000000) closing
[junit4] 2> 1093312 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:56027_solr
[junit4] 2> 1093312 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ElectionContext Canceling election /overseer_elect/election/95422824626585602-127.0.0.1:56027_solr-n_0000000000
[junit4] 2> 1093325 INFO (OverseerExitThread) [ ] o.a.s.c.Overseer I'm exiting , but I'm still the leader
[junit4] 2> 1093325 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95422824626585602-127.0.0.1:56027_solr-n_0000000001
[junit4] 2> 1093325 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56027_solr
[junit4] 2> 1093325 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 1093327 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422824626585602-127.0.0.1:56027_solr-n_0000000001) starting
[junit4] 2> 1093335 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1093355 INFO (OverseerCollectionConfigSetProcessor-95422824626585602-127.0.0.1:56027_solr-n_0000000001) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 1093356 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000001) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 1093356 INFO (zkCallback-2476-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1093356 INFO (zkCallback-2477-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1093359 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000001) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1093359 INFO (zkCallback-2476-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1093360 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000001) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://node1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 1093362 INFO (zkCallback-2476-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1094857 WARN (zkCallback-2477-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1094859 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56027 56027
[junit4] 2> 1095017 INFO (Thread-6683) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:56027 56027
[junit4] 2> 1095018 WARN (Thread-6683) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/live_nodes
[junit4] 2> 2 /solr/collections
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2>
[junit4] 2> 1095019 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testOverseerStatsReset
[junit4] 2> 1095019 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422824626585602-127.0.0.1:56027_solr-n_0000000001) closing
[junit4] 2> 1095020 INFO (OverseerStateUpdate-95422824626585602-127.0.0.1:56027_solr-n_0000000001) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:56027_solr
[junit4] 2> 1095024 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267)
[junit4] 2> 1095024 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testRemovalOfLastReplica
[junit4] 2> 1095025 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1095026 INFO (Thread-6684) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1095026 INFO (Thread-6684) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1095125 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer start zk server on port:60602
[junit4] 2> 1095125 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1095126 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1095128 INFO (zkCallback-2480-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@82b215b name:ZooKeeperConnection Watcher:127.0.0.1:60602 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1095128 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1095128 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1095129 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1095131 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1095131 INFO (zkCallback-2481-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@66d78cfe name:ZooKeeperConnection Watcher:127.0.0.1:60602 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1095131 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1095132 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1095132 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 1095134 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1095135 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1095136 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@11102c23 name:ZooKeeperConnection Watcher:127.0.0.1:60602/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1095136 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1095136 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1095136 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 1095138 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 1095139 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 1095140 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 1095141 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 1095142 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1095143 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1095144 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1095147 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1095147 INFO (zkCallback-2483-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2c171ec1 name:ZooKeeperConnection Watcher:127.0.0.1:60602/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1095147 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1095148 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1095148 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 1095150 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 1095151 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 1095152 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1095153 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95422824789639171-127.0.0.1:60602_solr-n_0000000000
[junit4] 2> 1095153 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60602_solr
[junit4] 2> 1095153 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 1095154 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422824789639171-127.0.0.1:60602_solr-n_0000000000) starting
[junit4] 2> 1095155 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 1095157 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 1095158 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 1095160 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 1095162 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 1095163 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 1095166 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1095167 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 1095167 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1095169 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1095171 INFO (OverseerCollectionConfigSetProcessor-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 1095174 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"create",
[junit4] 2> "name":"collection1",
[junit4] 2> "numShards":"2",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "maxShardsPerNode":"8"} current state version: 0
[junit4] 2> 1095174 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 1095175 INFO (zkCallback-2483-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095179 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1095179 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095230 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1095232 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node7",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core7",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095232 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node7",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core7",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095234 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node8",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core8",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095235 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node8",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core8",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095236 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core5",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095237 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core5",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095238 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 5, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core6",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095239 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core6",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095240 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 5, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core3",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095240 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core3",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095241 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 5, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core4",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095242 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core4",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095243 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 5, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095243 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard1",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095244 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 5, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core2",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 1095245 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "shard":"shard2",
[junit4] 2> "node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core2",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1095347 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095347 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095389 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node7",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core7",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095390 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node7",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core7",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095390 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 1095390 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 1095392 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node8",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core8",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095392 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node8",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core8",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095392 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node2 is already registered
[junit4] 2> 1095392 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard2 is already registered
[junit4] 2> 1095394 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 6, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core5",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095394 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core5",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095394 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node3 is already registered
[junit4] 2> 1095394 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 1095396 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 6, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core6",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095396 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core6",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095396 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node4 is already registered
[junit4] 2> 1095396 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard2 is already registered
[junit4] 2> 1095397 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 6, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core3",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095398 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core3",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095398 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node5 is already registered
[junit4] 2> 1095398 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 1095399 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 6, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core4",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095399 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core4",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095399 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node6 is already registered
[junit4] 2> 1095399 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard2 is already registered
[junit4] 2> 1095400 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 6, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095401 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095401 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node7 is already registered
[junit4] 2> 1095401 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 1095402 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 6, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core2",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 1095402 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core2",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1095402 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node8 is already registered
[junit4] 2> 1095402 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard2 is already registered
[junit4] 2> 1095504 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095504 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095543 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node7"} current state version: 3
[junit4] 2> 1095646 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095646 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095695 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node8"} current state version: 4
[junit4] 2> 1095797 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095797 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095846 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node5"} current state version: 5
[junit4] 2> 1095948 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095948 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1095997 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node6"} current state version: 6
[junit4] 2> 1096100 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096100 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096148 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node3"} current state version: 7
[junit4] 2> 1096254 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096254 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096299 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node4"} current state version: 8
[junit4] 2> 1096401 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096401 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096450 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node1"} current state version: 9
[junit4] 2> 1096555 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096556 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096601 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core_node_name":"core_node2"} current state version: 10
[junit4] 2> 1096703 INFO (zkCallback-2483-thread-2) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096704 INFO (zkCallback-2482-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1096753 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60602 60602
[junit4] 2> 1097177 INFO (OverseerCollectionConfigSetProcessor-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor.amILeader(OverseerTaskProcessor.java:354)
[junit4] 2> at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:171)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1097178 INFO (OverseerCollectionConfigSetProcessor-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor According to ZK I (id=95422824789639171-127.0.0.1:60602_solr-n_0000000000) am no longer a leader.
[junit4] 2> 1097212 INFO (Thread-6684) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60602 60602
[junit4] 2> 1097213 WARN (Thread-6684) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/live_nodes
[junit4] 2> 2 /solr/collections
[junit4] 2>
[junit4] 2> 1097213 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testRemovalOfLastReplica
[junit4] 2> 1097213 INFO (TEST-OverseerTest.testRemovalOfLastReplica-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422824789639171-127.0.0.1:60602_solr-n_0000000000) closing
[junit4] 2> 1097213 INFO (OverseerStateUpdate-95422824789639171-127.0.0.1:60602_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:60602_solr
[junit4] 2> 1097214 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267)
[junit4] 2> 1097216 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testShardAssignmentBigger
[junit4] 2> 1097217 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 1097217 INFO (Thread-6685) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1097217 INFO (Thread-6685) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 1097317 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer start zk server on port:52398
[junit4] 2> 1097317 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097318 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097320 INFO (zkCallback-2486-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6ca9e282 name:ZooKeeperConnection Watcher:127.0.0.1:52398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097320 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097320 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097321 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097322 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097323 INFO (zkCallback-2487-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@40cd4e1 name:ZooKeeperConnection Watcher:127.0.0.1:52398 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097323 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097323 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097323 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 1097326 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097327 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097327 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@59be3d28 name:ZooKeeperConnection Watcher:127.0.0.1:52398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097327 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097328 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097328 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 1097329 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 1097330 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 1097332 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 1097333 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 1097333 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097334 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097335 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3073bb82 name:ZooKeeperConnection Watcher:127.0.0.1:52398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097335 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097335 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097335 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 1097337 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 1097338 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 1097340 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 1097340 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95422824933228547-127.0.0.1:52398_solr-n_0000000000
[junit4] 2> 1097340 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52398_solr
[junit4] 2> 1097340 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 1097342 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422824933228547-127.0.0.1:52398_solr-n_0000000000) starting
[junit4] 2> 1097343 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 1097344 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 1097346 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 1097347 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 1097349 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 1097363 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 1097367 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 1097371 INFO (OverseerStateUpdate-95422824933228547-127.0.0.1:52398_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 1097371 INFO (OverseerStateUpdate-95422824933228547-127.0.0.1:52398_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1097372 INFO (OverseerStateUpdate-95422824933228547-127.0.0.1:52398_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1097379 INFO (OverseerCollectionConfigSetProcessor-95422824933228547-127.0.0.1:52398_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 1097379 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1097380 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1097381 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097382 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097383 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@f1a36d7 name:ZooKeeperConnection Watcher:127.0.0.1:52398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097383 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097383 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097384 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1097386 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 1097386 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node0
[junit4] 2> 1097388 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1097388 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1097388 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 1097389 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097389 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1097389 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1097391 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097409 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1097410 INFO (zkCallback-2493-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@63524ce4 name:ZooKeeperConnection Watcher:127.0.0.1:52398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097410 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097410 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097412 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1097413 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 1097413 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1
[junit4] 2> 1097415 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1097415 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097415 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1097416 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1097416 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1097416 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1097417 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1097421 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097422 INFO (zkCallback-2493-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1097422 INFO (zkCallback-2494-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@233e372 name:ZooKeeperConnection Watcher:127.0.0.1:52398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097423 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097423 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097423 INFO (zkCallback-2493-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
[junit4] 2> 1097424 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1097425 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (2)
[junit4] 2> 1097426 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node2
[junit4] 2> 1097427 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [2])
[junit4] 2> 1097427 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [2])
[junit4] 2> 1097427 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [2])
[junit4] 2> 1097427 INFO (zkCallback-2493-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [2])
[junit4] 2> 1097428 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1097428 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1097428 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1097428 INFO (zkCallback-2493-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1097428 INFO (zkCallback-2494-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [2])
[junit4] 2> 1097429 INFO (zkCallback-2494-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (2) -> (3)
[junit4] 2> 1097429 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097431 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097431 INFO (zkCallback-2495-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@c213453 name:ZooKeeperConnection Watcher:127.0.0.1:52398/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1097431 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 1097432 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 1097433 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 1097434 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (3)
[junit4] 2> 1097435 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node3
[junit4] 2> 1097436 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3])
[junit4] 2> 1097436 INFO (zkCallback-2493-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3])
[junit4] 2> 1097437 INFO (zkCallback-2494-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3])
[junit4] 2> 1097437 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3])
[junit4] 2> 1097437 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3])
[junit4] 2> 1097437 INFO (zkCallback-2488-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1097437 INFO (zkCallback-2494-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1097437 INFO (zkCallback-2492-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1097437 INFO (zkCallback-2489-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1097437 INFO (zkCallback-2493-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 1097438 INFO (zkCallback-2495-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [3])
[junit4] 2> 1097438 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 1097471 INFO (TEST-OverseerTest.testShardAssignmentBigger-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 1097472 INFO (zkCallback-2495-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (3) -> (4)
[junit4] 2> 109
[...truncated too long message...]
6 INFO (OverseerStateUpdate-95422831645360133-127.0.0.1:51412_solr-n_0000000001) [ ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://node1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current state version: 3
[junit4] 2> 1202507 INFO (OverseerStateUpdate-95422831645360133-127.0.0.1:51412_solr-n_0000000001) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"deletecore",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1"} current state version: 3
[junit4] 2> 1202610 INFO (zkCallback-2594-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1202610 INFO (zkCallback-2589-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1202610 INFO (zkCallback-2590-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1203111 INFO (zkCallback-2589-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1203111 INFO (zkCallback-2594-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1203111 INFO (zkCallback-2590-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1203112 INFO (zkCallback-2594-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1203112 INFO (zkCallback-2589-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1204611 WARN (zkCallback-2590-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 1204613 INFO (TEST-OverseerTest.testOverseerFailure-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51412 51412
[junit4] 2> 1204836 INFO (Thread-6695) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:51412 51412
[junit4] 2> 1204837 WARN (Thread-6695) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 4 /solr/live_nodes
[junit4] 2> 4 /solr/collections
[junit4] 2> 2 /solr/overseer/queue
[junit4] 2> 2 /solr/overseer/collection-queue-work
[junit4] 2> 2 /solr/overseer/queue-work
[junit4] 2>
[junit4] 2> 1204837 INFO (TEST-OverseerTest.testOverseerFailure-seed#[58ADC6D3DE86652B]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testOverseerFailure
[junit4] 2> 1204837 INFO (TEST-OverseerTest.testOverseerFailure-seed#[58ADC6D3DE86652B]) [ ] o.a.s.c.Overseer Overseer (id=95422831645360133-127.0.0.1:51412_solr-n_0000000001) closing
[junit4] 2> 1204838 INFO (OverseerStateUpdate-95422831645360133-127.0.0.1:51412_solr-n_0000000001) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:51412_solr
[junit4] 2> 1204839 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267)
[junit4] 2> 1207840 INFO (SUITE-OverseerTest-seed#[58ADC6D3DE86652B]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/build/solr-core/test/J1/temp/solr.cloud.OverseerTest_58ADC6D3DE86652B-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, sim=RandomSimilarity(queryNorm=false,coord=crazy): {}, locale=fr-CH, timezone=SystemV/MST7
[junit4] 2> NOTE: Linux 3.13.0-52-generic amd64/Oracle Corporation 1.8.0_66 (64-bit)/cpus=4,threads=1,free=181405920,total=515899392
[junit4] 2> NOTE: All tests run in this JVM: [TestSimpleQParserPlugin, TestSimpleTrackingShardHandler, TestSurroundQueryParser, TimeZoneUtilsTest, MultiThreadedOCPTest, VersionInfoTest, TestCodecSupport, TestLFUCache, PKIAuthenticationIntegrationTest, SpellingQueryConverterTest, HdfsWriteToMultipleCollectionsTest, BasicFunctionalityTest, TestUseDocValuesAsStored2, TestPivotHelperCode, SpatialFilterTest, CSVRequestHandlerTest, SpellCheckCollatorTest, DeleteLastCustomShardedReplicaTest, TestSolrQueryParserResource, TestConfigSetsAPI, TestClassicSimilarityFactory, SharedFSAutoReplicaFailoverTest, TriLevelCompositeIdRoutingTest, ParsingFieldUpdateProcessorsTest, TestDistributedGrouping, TestShortCircuitedRequests, TestQuerySenderNoQuery, CollectionsAPIAsyncDistributedZkTest, CdcrReplicationHandlerTest, TestBinaryResponseWriter, SolrInfoMBeanTest, TestWordDelimiterFilterFactory, TestQuerySenderListener, ZkControllerTest, TestSchemaNameResource, FastVectorHighlighterTest, CloneFieldUpdateProcessorFactoryTest, TermVectorComponentTest, TestSolrDeletionPolicy2, TestOrdValues, NumericFieldsTest, OpenExchangeRatesOrgProviderTest, TestFileDictionaryLookup, TestCopyFieldCollectionResource, SolrCmdDistributorTest, TestFieldTypeCollectionResource, SpatialRPTFieldTypeTest, UpdateRequestProcessorFactoryTest, TestRecoveryHdfs, TestCursorMarkWithoutUniqueKey, DirectSolrSpellCheckerTest, TestCloudManagedSchemaConcurrent, SimplePostToolTest, UniqFieldsUpdateProcessorFactoryTest, GraphQueryTest, TestSolrConfigHandlerCloud, SolrIndexSplitterTest, AsyncMigrateRouteKeyTest, TestManagedSynonymFilterFactory, DistributedExpandComponentTest, TestManagedSchema, HdfsBasicDistributedZkTest, DistributedQueryComponentCustomSortTest, RulesTest, SSLMigrationTest, ReplicationFactorTest, TestSchemaVersionResource, MoreLikeThisHandlerTest, ChaosMonkeySafeLeaderTest, SpellCheckComponentTest, DistributedTermsComponentTest, ExitableDirectoryReaderTest, NotRequiredUniqueKeyTest, BlockCacheTest, TestCoreContainer, SchemaVersionSpecificBehaviorTest, TestCloudInspectUtil, TestPartialUpdateDeduplication, SolrCloudExampleTest, TestCryptoKeys, TestXmlQParser, SolrIndexConfigTest, SecurityConfHandlerTest, TestEmbeddedSolrServerConstructors, OverseerTaskQueueTest, ChaosMonkeyNothingIsSafeTest, TestReplicaProperties, SortSpecParsingTest, ClusterStateUpdateTest, BigEndianAscendingWordDeserializerTest, UUIDUpdateProcessorFallbackTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OverseerTest]
[junit4] Completed [297/579 (1!)] on J1 in 115.38s, 13 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 894 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/build.xml:740: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/build.xml:684: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/common-build.xml:524: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/lucene/common-build.xml:1457: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/lucene/common-build.xml:1014: There were test failures: 579 suites (8 ignored), 2331 tests, 1 error, 88 ignored (72 assumptions) [seed: 58ADC6D3DE86652B]
Total time: 59 minutes 9 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
[JENKINS] Lucene-Solr-Tests-trunk-Java8 - Build # 848 - Still
Failing
Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-trunk-Java8/848/
1 tests failed.
FAILED: org.apache.solr.cloud.OverseerTest.testShardLeaderChange
Error Message:
Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
Stack Trace:
org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
at __randomizedtesting.SeedInfo.seed([C4F17363A5436D83:1AA2F494BFDB9872]:0)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:212)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:173)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:138)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:310)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:219)
at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:181)
at org.apache.solr.cloud.OverseerTest.testShardLeaderChange(OverseerTest.java:841)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists
at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
at org.apache.zookeeper.ZooKeeper.multiInternal(ZooKeeper.java:949)
at org.apache.zookeeper.ZooKeeper.multi(ZooKeeper.java:915)
at org.apache.solr.common.cloud.SolrZkClient$11.execute(SolrZkClient.java:577)
at org.apache.solr.common.cloud.SolrZkClient$11.execute(SolrZkClient.java:574)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
at org.apache.solr.common.cloud.SolrZkClient.multi(SolrZkClient.java:574)
at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:195)
at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:49)
at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:42)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:178)
... 45 more
Build Log:
[...truncated 11278 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/build/solr-core/test/J1/temp/solr.cloud.OverseerTest_C4F17363A5436D83-001/init-core-data-001
[junit4] 2> 926530 INFO (SUITE-OverseerTest-seed#[C4F17363A5436D83]-worker) [ ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false)
[junit4] 2> 926530 INFO (SUITE-OverseerTest-seed#[C4F17363A5436D83]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore
[junit4] 2> 926531 INFO (SUITE-OverseerTest-seed#[C4F17363A5436D83]-worker) [ ] o.a.s.SolrTestCaseJ4 ####initCore end
[junit4] 2> 926533 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testShardAssignment
[junit4] 2> 926533 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 926533 INFO (Thread-5303) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 926533 INFO (Thread-5303) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 926633 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer start zk server on port:34647
[junit4] 2> 926633 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 926635 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 926637 INFO (zkCallback-1559-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7b3cded name:ZooKeeperConnection Watcher:127.0.0.1:34647 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 926637 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 926638 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 926639 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15304bd277b0000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 926639 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 926640 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 926646 INFO (zkCallback-1560-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@34a6894a name:ZooKeeperConnection Watcher:127.0.0.1:34647 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 926646 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 926647 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 926647 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 926649 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 926649 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 926650 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4bbea1a5 name:ZooKeeperConnection Watcher:127.0.0.1:34647/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 926650 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 926651 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 926651 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 926652 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 926653 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 926654 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 926655 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 926655 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 926660 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 926660 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@31da0c2b name:ZooKeeperConnection Watcher:127.0.0.1:34647/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 926660 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 926661 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 926661 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 926662 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 926663 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 926664 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 926665 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95425227562614787-127.0.0.1:34647_solr-n_0000000000
[junit4] 2> 926665 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:34647_solr
[junit4] 2> 926665 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 926666 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425227562614787-127.0.0.1:34647_solr-n_0000000000) starting
[junit4] 2> 926667 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 926668 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 926670 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 926671 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 926673 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 926674 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 926677 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 926677 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 926678 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 926679 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 926684 INFO (OverseerCollectionConfigSetProcessor-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 926684 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 926685 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 926686 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 926687 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 926688 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6c56a5fe name:ZooKeeperConnection Watcher:127.0.0.1:34647/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 926688 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 926688 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 926689 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 926690 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 926691 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1
[junit4] 2> 926692 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 926692 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 926693 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 926696 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 926696 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 926697 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 926698 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 0
[junit4] 2> 926699 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 926699 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 926699 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 926700 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 926700 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 926700 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 927198 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 927200 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 927200 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 927200 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 927203 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard2/election/95425227562614788-127.0.0.1_core1-n_0000000000
[junit4] 2> 927204 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard2
[junit4] 2> 927206 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard2/leader after winning as /collections/collection1/leader_elect/shard2/election/95425227562614788-127.0.0.1_core1-n_0000000000
[junit4] 2> 927208 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://127.0.0.1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 927210 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 1
[junit4] 2> 927211 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 927211 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3
[junit4] 2> 927211 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard3
[junit4] 2> 927314 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 927314 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 927314 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 927709 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 927713 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard3/election/95425227562614788-127.0.0.1_core2-n_0000000000
[junit4] 2> 927714 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard3
[junit4] 2> 927716 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard3/leader after winning as /collections/collection1/leader_elect/shard3/election/95425227562614788-127.0.0.1_core2-n_0000000000
[junit4] 2> 927718 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://127.0.0.1/solr/",
[junit4] 2> "core":"core2",
[junit4] 2> "state":"active"} current state version: 2
[junit4] 2> 927720 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 2
[junit4] 2> 927721 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 927721 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3
[junit4] 2> 927721 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 927823 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 927823 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 927823 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 928220 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 928224 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95425227562614788-127.0.0.1_core3-n_0000000000
[junit4] 2> 928225 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1
[junit4] 2> 928227 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard1/leader after winning as /collections/collection1/leader_elect/shard1/election/95425227562614788-127.0.0.1_core3-n_0000000000
[junit4] 2> 928230 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://127.0.0.1/solr/",
[junit4] 2> "core":"core3",
[junit4] 2> "state":"active"} current state version: 3
[junit4] 2> 928232 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core4",
[junit4] 2> "core_node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 3
[junit4] 2> 928232 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core4",
[junit4] 2> "core_node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 928232 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3
[junit4] 2> 928232 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
[junit4] 2> 928334 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 928334 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 928335 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 928731 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 928735 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard2/election/95425227562614788-127.0.0.1_core4-n_0000000001
[junit4] 2> 928736 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Watching path /collections/collection1/leader_elect/shard2/election/95425227562614788-127.0.0.1_core1-n_0000000000 to know if I could be the leader
[junit4] 2> 928738 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core5",
[junit4] 2> "core_node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 4
[junit4] 2> 928739 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core5",
[junit4] 2> "core_node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 928739 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3
[junit4] 2> 928739 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard3
[junit4] 2> 928842 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 928842 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 928842 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 929237 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 929241 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard3/election/95425227562614788-127.0.0.1_core5-n_0000000001
[junit4] 2> 929242 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Watching path /collections/collection1/leader_elect/shard3/election/95425227562614788-127.0.0.1_core2-n_0000000000 to know if I could be the leader
[junit4] 2> 929245 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core6",
[junit4] 2> "core_node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"} current state version: 5
[junit4] 2> 929246 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core6",
[junit4] 2> "core_node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 929246 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Collection already exists with numShards=3
[junit4] 2> 929246 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 929350 INFO (zkCallback-1561-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 929350 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 929350 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 929744 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 929748 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95425227562614788-127.0.0.1_core6-n_0000000001
[junit4] 2> 929749 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Watching path /collections/collection1/leader_elect/shard1/election/95425227562614788-127.0.0.1_core3-n_0000000000 to know if I could be the leader
[junit4] 2> 929750 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 929750 INFO (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 929751 INFO (zkCallback-1562-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 929751 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15304bd277b0004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 929751 WARN (zkCallback-1565-thread-2) [ ] o.a.s.c.LeaderElector
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:336)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:333)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:333)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:59)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:351)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3$1.run(SolrZkClient.java:269)
[junit4] 2> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[junit4] 2> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1.run(ExecutorUtil.java:231)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 929751 ERROR (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]-EventThread) [ ] o.a.z.ClientCnxn Error while calling watcher
[junit4] 2> java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1@2bb1b1f2 rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@6eb32ae0[Shutting down, pool size = 2, active threads = 2, queued tasks = 0, completed tasks = 9]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:214)
[junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:266)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[junit4] 2> 929752 ERROR (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]-EventThread) [ ] o.a.z.ClientCnxn Error while calling watcher
[junit4] 2> java.util.concurrent.RejectedExecutionException: Task org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$1@5f170075 rejected from org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor@6eb32ae0[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 10]
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2047)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:823)
[junit4] 2> at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1369)
[junit4] 2> at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.execute(ExecutorUtil.java:214)
[junit4] 2> at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:112)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$3.process(SolrZkClient.java:266)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
[junit4] 2> 931251 WARN (zkCallback-1565-thread-1) [ ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
[junit4] 2> 931253 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34647 34647
[junit4] 2> 931374 INFO (Thread-5303) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:34647 34647
[junit4] 2> 931375 WARN (Thread-5303) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> 931375 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testShardAssignment
[junit4] 2> 931375 INFO (TEST-OverseerTest.testShardAssignment-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425227562614787-127.0.0.1:34647_solr-n_0000000000) closing
[junit4] 2> 931375 INFO (OverseerStateUpdate-95425227562614787-127.0.0.1:34647_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:34647_solr
[junit4] 2> 931377 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267)
[junit4] 2> 931379 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testExternalClusterStateChangeBehavior
[junit4] 2> 931379 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 931380 INFO (Thread-5304) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 931380 INFO (Thread-5304) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 931480 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer start zk server on port:57155
[junit4] 2> 931480 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 931480 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 931483 INFO (zkCallback-1566-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2d65dc21 name:ZooKeeperConnection Watcher:127.0.0.1:57155/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 931483 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 931483 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 931484 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 931484 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 931485 INFO (zkCallback-1567-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7e379fb name:ZooKeeperConnection Watcher:127.0.0.1:57155 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 931485 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 931485 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 931486 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 931486 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 931488 INFO (zkCallback-1568-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4c6f890b name:ZooKeeperConnection Watcher:127.0.0.1:57155 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 931488 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 931488 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 931488 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 931490 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 931491 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 931493 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 931498 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 931499 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 931500 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 931502 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931502 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 931503 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 931504 INFO (zkCallback-1569-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1713deb6 name:ZooKeeperConnection Watcher:127.0.0.1:57155/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 931504 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 931504 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 931504 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 931506 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 931507 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 931508 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 931509 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95425227880202243-127.0.0.1:57155_solr-n_0000000000
[junit4] 2> 931509 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:57155_solr
[junit4] 2> 931509 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 931511 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425227880202243-127.0.0.1:57155_solr-n_0000000000) starting
[junit4] 2> 931512 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 931514 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 931515 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 931517 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 931519 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 931521 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 931524 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 931525 INFO (OverseerCollectionConfigSetProcessor-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 931525 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 931526 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 931527 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931527 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931528 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"down"} current state version: 0
[junit4] 2> 931529 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"down"}
[junit4] 2> 931529 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 931530 INFO (zkCallback-1566-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931530 INFO (zkCallback-1569-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931579 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931580 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"} current state version: 1
[junit4] 2> 931581 INFO (zkCallback-1566-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931581 INFO (zkCallback-1569-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931581 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 931581 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 931581 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 931583 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"} current state version: 1
[junit4] 2> 931583 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 931584 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 931584 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 931584 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931585 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
[junit4] 2> "operation":"create",
[junit4] 2> "name":"test",
[junit4] 2> "numShards":"1",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "stateFormat":"2"} current state version: 1
[junit4] 2> 931585 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: test
[junit4] 2> 931586 WARN (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer Bad version writing to ZK using compare-and-set, will force refresh cluster state
[junit4] 2> org.apache.zookeeper.KeeperException$BadVersionException: KeeperErrorCode = BadVersion for /clusterstate.json
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:115)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.setData(ZooKeeper.java:1270)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:370)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$8.execute(SolrZkClient.java:367)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.setData(SolrZkClient.java:367)
[junit4] 2> at org.apache.solr.cloud.overseer.ZkStateWriter.writePendingUpdates(ZkStateWriter.java:252)
[junit4] 2> at org.apache.solr.cloud.overseer.ZkStateWriter.enqueueUpdate(ZkStateWriter.java:119)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.processQueueItem(Overseer.java:293)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:216)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 931587 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931588 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: workQueueSize: 2, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 931588 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 931588 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 931588 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 931589 INFO (zkCallback-1566-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931589 INFO (zkCallback-1569-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931590 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: workQueueSize: 2, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 931591 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"c1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 931591 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator node=core_node1 is already registered
[junit4] 2> 931591 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator shard=shard1 is already registered
[junit4] 2> 931591 INFO (zkCallback-1566-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931591 INFO (zkCallback-1569-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [0])
[junit4] 2> 931592 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
[junit4] 2> "operation":"create",
[junit4] 2> "name":"test",
[junit4] 2> "numShards":"1",
[junit4] 2> "replicationFactor":"1",
[junit4] 2> "stateFormat":"2"} current state version: 4
[junit4] 2> 931592 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: test
[junit4] 2> 931594 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
[junit4] 2> "operation":"createshard",
[junit4] 2> "collection":"test",
[junit4] 2> "shard":"x",
[junit4] 2> "replicationFactor":"1"} current state version: 4
[junit4] 2> 931596 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"test",
[junit4] 2> "shard":"x",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"down"} current state version: 4
[junit4] 2> 931596 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.SliceMutator createReplica() {
[junit4] 2> "operation":"addreplica",
[junit4] 2> "collection":"test",
[junit4] 2> "shard":"x",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"down"}
[junit4] 2> 931635 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931687 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931698 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/test/state.json
[junit4] 2> 931738 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931740 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 931744 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57155 57155
[junit4] 2> 931938 INFO (Thread-5304) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57155 57155
[junit4] 2> 931939 WARN (Thread-5304) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 2 /solr/live_nodes
[junit4] 2> 2 /solr/collections
[junit4] 2>
[junit4] 2> 931939 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testExternalClusterStateChangeBehavior
[junit4] 2> 931939 INFO (TEST-OverseerTest.testExternalClusterStateChangeBehavior-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425227880202243-127.0.0.1:57155_solr-n_0000000000) closing
[junit4] 2> 931939 INFO (OverseerStateUpdate-95425227880202243-127.0.0.1:57155_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57155_solr
[junit4] 2> 931940 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267)
[junit4] 2> 931942 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.SolrTestCaseJ4 ###Starting testOverseerStatsReset
[junit4] 2> 931943 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
[junit4] 2> 931943 INFO (Thread-5305) [ ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 931943 INFO (Thread-5305) [ ] o.a.s.c.ZkTestServer Starting server
[junit4] 2> 932043 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer start zk server on port:52673
[junit4] 2> 932043 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 932044 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 932046 INFO (zkCallback-1572-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6d2ab476 name:ZooKeeperConnection Watcher:127.0.0.1:52673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 932046 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 932046 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 932047 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 932048 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 932049 INFO (zkCallback-1573-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3e0636ec name:ZooKeeperConnection Watcher:127.0.0.1:52673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 932049 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 932049 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 932049 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /solr
[junit4] 2> 932051 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 932051 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 932052 INFO (zkCallback-1574-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@ae84f6b name:ZooKeeperConnection Watcher:127.0.0.1:52673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 932052 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 932053 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 932053 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
[junit4] 2> 932054 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections
[junit4] 2> 932055 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
[junit4] 2> 932057 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
[junit4] 2> 932058 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /security.json
[junit4] 2> 932059 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 932060 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 932061 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
[junit4] 2> 932061 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
[junit4] 2> 932062 INFO (zkCallback-1575-thread-1) [ ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7061bbaf name:ZooKeeperConnection Watcher:127.0.0.1:52673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 932062 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
[junit4] 2> 932062 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
[junit4] 2> 932064 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper...
[junit4] 2> 932065 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 932066 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/node1
[junit4] 2> 932067 INFO (zkCallback-1575-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 932067 INFO (zkCallback-1574-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
[junit4] 2> 932068 INFO (zkCallback-1574-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 932069 INFO (zkCallback-1575-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
[junit4] 2> 932069 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.u.UpdateShardHandler Creating UpdateShardHandler HTTP client with params: socketTimeout=600000&connTimeout=60000&retry=true
[junit4] 2> 932070 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
[junit4] 2> 932071 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
[junit4] 2> 932072 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=null) closing
[junit4] 2> 932073 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95425227917164546-127.0.0.1:52673_solr-n_0000000000
[junit4] 2> 932073 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52673_solr
[junit4] 2> 932073 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 932074 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425227917164546-127.0.0.1:52673_solr-n_0000000000) starting
[junit4] 2> 932075 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
[junit4] 2> 932077 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
[junit4] 2> 932079 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
[junit4] 2> 932081 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
[junit4] 2> 932082 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
[junit4] 2> 932084 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
[junit4] 2> 932087 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 932088 INFO (OverseerCollectionConfigSetProcessor-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 932089 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 932091 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
[junit4] 2> 932092 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"} current state version: 0
[junit4] 2> 932092 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 932092 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
[junit4] 2> 932092 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
[junit4] 2> 932094 INFO (zkCallback-1574-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 932094 INFO (zkCallback-1575-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 932589 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 932591 INFO (zkCallback-1574-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 932591 INFO (zkCallback-1575-thread-1) [ ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
[junit4] 2> 932593 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /collections/collection1/leader_elect/shard1/election/95425227917164547-node1_core1-n_0000000000
[junit4] 2> 932594 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/leaders/shard1
[junit4] 2> 932596 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection1/leaders/shard1/leader after winning as /collections/collection1/leader_elect/shard1/election/95425227917164547-node1_core1-n_0000000000
[junit4] 2> 932597 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425227917164546-127.0.0.1:52673_solr-n_0000000000) closing
[junit4] 2> 932597 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ElectionContext Canceling election /overseer_elect/election/95425227917164546-127.0.0.1:52673_solr-n_0000000000
[junit4] 2> 932597 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:52673_solr
[junit4] 2> 932600 INFO (OverseerExitThread) [ ] o.a.s.c.Overseer I'm exiting , but I'm still the leader
[junit4] 2> 932600 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/95425227917164546-127.0.0.1:52673_solr-n_0000000001
[junit4] 2> 932601 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52673_solr
[junit4] 2> 932601 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
[junit4] 2> 932602 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425227917164546-127.0.0.1:52673_solr-n_0000000001) starting
[junit4] 2> 932608 INFO (TEST-OverseerTest.testOverseerStatsReset-seed#[C4F17363A5436D83]) [ ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
[junit4] 2> 932608 INFO (OverseerCollectionConfigSetProcessor-95425227917164546-127.0.0.1:52673_solr-n_0000000001) [ ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
[junit4] 2> 932608 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000001) [ ] o.a.s.c.Overseer Starting to work on the main queue
[junit4] 2> 932609 INFO (zkCallback-1574-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 932609 INFO (zkCallback-1575-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 932610 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000001) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 932611 INFO (zkCallback-1574-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
[junit4] 2> 932611 INFO (OverseerStateUpdate-95425227917164546-127.0.0.1:52673_solr-n_0000000001) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "base_url":"http://node1/solr/",
[junit4] 2> "core":"core1",
[junit4] 2> "state":"active"} current
[...truncated too long message...]
aderElector Joined leadership election with path: /collections/collection2/leader_elect/shard1/election/95425234167005188-127.0.0.1_core3-n_0000000000
[junit4] 2> 1030697 INFO (TEST-OverseerTest.testBadQueueItem-seed#[C4F17363A5436D83]) [ ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2/leaders/shard1
[junit4] 2> 1030699 INFO (TEST-OverseerTest.testBadQueueItem-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ShardLeaderElectionContextBase Creating leader registration node /collections/collection2/leaders/shard1/leader after winning as /collections/collection2/leader_elect/shard1/election/95425234167005188-127.0.0.1_core3-n_0000000000
[junit4] 2> 1030702 INFO (OverseerStateUpdate-95425234167005187-127.0.0.1:57721_solr-n_0000000000) [ ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
[junit4] 2> "operation":"leader",
[junit4] 2> "shard":"shard1",
[junit4] 2> "collection":"collection2",
[junit4] 2> "base_url":"http://127.0.0.1/solr/",
[junit4] 2> "core":"core3",
[junit4] 2> "state":"active"} current state version: 7
[junit4] 2> 1030806 INFO (zkCallback-1675-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1030806 INFO (zkCallback-1674-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1030806 INFO (zkCallback-1678-thread-1) [ ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1030854 INFO (zkCallback-1678-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1030854 INFO (zkCallback-1675-thread-1) [ ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
[junit4] 2> 1030854 INFO (zkCallback-1678-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1030854 INFO (zkCallback-1675-thread-1) [ ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (0)
[junit4] 2> 1030855 WARN (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [ ] o.a.z.s.NIOServerCnxn caught end of stream exception
[junit4] 2> EndOfStreamException: Unable to read additional data from client sessionid 0x15304beb1220004, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:745)
[junit4] 2> 1030858 INFO (TEST-OverseerTest.testBadQueueItem-seed#[C4F17363A5436D83]) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57721 57721
[junit4] 2> 1031017 INFO (Thread-5315) [ ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:57721 57721
[junit4] 2> 1031019 WARN (Thread-5315) [ ] o.a.s.c.ZkTestServer Watch limit violations:
[junit4] 2> Maximum concurrent create/delete watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/aliases.json
[junit4] 2>
[junit4] 2> Maximum concurrent data watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/clusterstate.json
[junit4] 2>
[junit4] 2> Maximum concurrent children watches above limit:
[junit4] 2>
[junit4] 2> 3 /solr/live_nodes
[junit4] 2> 3 /solr/collections
[junit4] 2>
[junit4] 2> 1031021 INFO (TEST-OverseerTest.testBadQueueItem-seed#[C4F17363A5436D83]) [ ] o.a.s.SolrTestCaseJ4 ###Ending testBadQueueItem
[junit4] 2> 1031021 INFO (TEST-OverseerTest.testBadQueueItem-seed#[C4F17363A5436D83]) [ ] o.a.s.c.Overseer Overseer (id=95425234167005187-127.0.0.1:57721_solr-n_0000000000) closing
[junit4] 2> 1031021 INFO (OverseerStateUpdate-95425234167005187-127.0.0.1:57721_solr-n_0000000000) [ ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57721_solr
[junit4] 2> 1031022 ERROR (OverseerExitThread) [ ] o.a.s.c.Overseer could not read the data
[junit4] 2> org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/leader
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:350)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.checkIfIamStillLeader(Overseer.java:308)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.access$300(Overseer.java:89)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater$2.run(Overseer.java:267)
[junit4] 2> 1034024 INFO (SUITE-OverseerTest-seed#[C4F17363A5436D83]-worker) [ ] o.a.s.SolrTestCaseJ4 ###deleteCore
[junit4] 2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/build/solr-core/test/J1/temp/solr.cloud.OverseerTest_C4F17363A5436D83-001
[junit4] 2> NOTE: test params are: codec=Asserting(Lucene60): {}, docValues:{}, sim=ClassicSimilarity, locale=th-TH-u-nu-thai-x-lvariant-TH, timezone=Asia/Amman
[junit4] 2> NOTE: Linux 3.13.0-52-generic amd64/Oracle Corporation 1.8.0_66 (64-bit)/cpus=4,threads=1,free=159118336,total=526909440
[junit4] 2> NOTE: All tests run in this JVM: [TestManagedSynonymFilterFactory, TestReqParamsAPI, ReplicationFactorTest, TestDocumentBuilder, TestHashPartitioner, DistributedTermsComponentTest, TestRecoveryHdfs, SortSpecParsingTest, SuggestComponentTest, TestSimpleTrackingShardHandler, TestSolrCoreProperties, TestCopyFieldCollectionResource, RulesTest, TestPivotHelperCode, TestWordDelimiterFilterFactory, ShardRoutingTest, TestShortCircuitedRequests, TestBlobHandler, BlockJoinFacetDistribTest, TestSystemIdResolver, HdfsCollectionsAPIDistributedZkTest, CoreAdminRequestStatusTest, TestQueryUtils, TestTrie, OverseerTaskQueueTest, TermsComponentTest, TestSchemaNameResource, ZkCLITest, GraphQueryTest, CdcrReplicationHandlerTest, TestCloudInspectUtil, CollectionsAPIDistributedZkTest, TestReplicaProperties, VersionInfoTest, SolrCloudExampleTest, ZkSolrClientTest, TestDynamicFieldResource, NumericFieldsTest, FileUtilsTest, ExitableDirectoryReaderTest, DeleteLastCustomShardedReplicaTest, OpenCloseCoreStressTest, OverseerCollectionConfigSetProcessorTest, HdfsBasicDistributedZkTest, TestSolrConfigHandlerCloud, TestCryptoKeys, TestFieldTypeCollectionResource, UUIDUpdateProcessorFallbackTest, TestDocBasedVersionConstraints, TestInfoStreamLogging, TestQuerySenderListener, TestFastLRUCache, TermVectorComponentTest, TestRandomCollapseQParserPlugin, TestRTimerTree, TestSolrDynamicMBean, SuggesterFSTTest, DistributedMLTComponentTest, TestCursorMarkWithoutUniqueKey, TestSolrDeletionPolicy2, DirectSolrSpellCheckerTest, TestRawResponseWriter, MBeansHandlerTest, ClusterStateUpdateTest, ShardRoutingCustomTest, LeaderInitiatedRecoveryOnCommitTest, HLLUtilTest, TestPerFieldSimilarityClassic, TestMacros, TestManagedSchema, TestManagedSchemaFieldResource, ExplicitHLLTest, CursorMarkTest, TestSurroundQueryParser, TestSolrQueryParserResource, ZkStateReaderTest, TestSchemaVersionResource, ChaosMonkeySafeLeaderTest, SimplePostToolTest, NotRequiredUniqueKeyTest, HighlighterMaxOffsetTest, AnalysisErrorHandlingTest, TestExactStatsCache, CollectionReloadTest, TestSimpleQParserPlugin, TestJsonRequest, TestPartialUpdateDeduplication, TestFieldSortValues, MultiTermTest, ConnectionManagerTest, SSLMigrationTest, BasicFunctionalityTest, OpenExchangeRatesOrgProviderTest, OverseerTest]
[junit4] Completed [295/579 (1!)] on J1 in 107.51s, 13 tests, 1 error, 1 skipped <<< FAILURES!
[...truncated 942 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/build.xml:740: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/build.xml:684: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/solr/common-build.xml:524: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/lucene/common-build.xml:1457: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-Tests-trunk-Java8/lucene/common-build.xml:1014: There were test failures: 579 suites (8 ignored), 2331 tests, 1 error, 88 ignored (72 assumptions) [seed: C4F17363A5436D83]
Total time: 60 minutes 53 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any