You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/02/06 01:23:16 UTC
[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.6.0_45) - Build # 9270
- Failure!
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/9270/
Java: 64bit/jdk1.6.0_45 -XX:+UseCompressedOops -XX:+UseParallelGC
1 tests failed.
REGRESSION: org.apache.solr.cloud.OverseerTest.testOverseerFailure
Error Message:
KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
Stack Trace:
org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
at __randomizedtesting.SeedInfo.seed([3A671B75C53DB535:3E6F9486D7985A14]:0)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:428)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:425)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
at java.lang.reflect.Method.invoke(Method.java:597)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
at java.lang.Thread.run(Thread.java:662)
Build Log:
[...truncated 9910 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> 719331 T3483 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-OverseerTest-1391645531514
[junit4] 2> 719332 T3483 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 719334 T3483 oas.SolrTestCaseJ4.setUp ###Starting testStateChange
[junit4] 2> 719335 T3483 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 719335 T3484 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 719435 T3483 oasc.ZkTestServer.run start zk server on port:34134
[junit4] 2> 719436 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 719490 T3490 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11532363 name:ZooKeeperConnection Watcher:127.0.0.1:34134/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 719490 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 719491 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 719494 T3492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a86ad45 name:ZooKeeperConnection Watcher:127.0.0.1:34134 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 719494 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 719499 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 719501 T3494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6014e924 name:ZooKeeperConnection Watcher:127.0.0.1:34134 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 719502 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 719502 T3483 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 719510 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 719513 T3483 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 719516 T3483 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 719518 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 719522 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 719524 T3496 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5083b08c name:ZooKeeperConnection Watcher:127.0.0.1:34134/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 719524 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 719525 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 719528 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 719533 T3483 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34134_solr
[junit4] 2> 719533 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 719536 T3483 oasc.Overseer.start Overseer (id=91202881553629187-127.0.0.1:34134_solr-n_0000000000) starting
[junit4] 2> 719538 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 719543 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 719548 T3483 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 719552 T3498 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 719554 T3498 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 719555 T3497 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 719556 T3483 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 719559 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 719560 T3497 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 719560 T3497 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 719563 T3496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 719564 T3490 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 719609 T3483 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 719613 T3496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 719617 T3497 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 719618 T3497 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 719619 T3497 oasc.Overseer$ClusterStateUpdater.updateState node=core_node1 is already registered
[junit4] 2> 719619 T3497 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 719625 T3496 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 719728 T3490 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 719768 T3483 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:34134 34134
[junit4] 2> 719948 T3483 oas.SolrTestCaseJ4.tearDown ###Ending testStateChange
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 719958 T3483 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignment
[junit4] 2> 719959 T3483 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 719959 T3499 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 720059 T3483 oasc.ZkTestServer.run start zk server on port:40089
[junit4] 2> 720060 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 720064 T3505 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3933840e name:ZooKeeperConnection Watcher:127.0.0.1:40089 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 720064 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 720083 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 720088 T3507 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@151bf7d6 name:ZooKeeperConnection Watcher:127.0.0.1:40089 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 720089 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 720089 T3483 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 720093 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 720094 T3509 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e3ea5e8 name:ZooKeeperConnection Watcher:127.0.0.1:40089/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 720095 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 720095 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 720098 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 720100 T3511 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7bebb374 name:ZooKeeperConnection Watcher:127.0.0.1:40089/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 720100 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 720101 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 720104 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 720110 T3483 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:40089_solr
[junit4] 2> 720111 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 720114 T3483 oasc.Overseer.start Overseer (id=91202881594720259-127.0.0.1:40089_solr-n_0000000000) starting
[junit4] 2> 720116 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 720121 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 720126 T3483 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 720129 T3513 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 720130 T3483 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 720130 T3513 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 720132 T3512 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 720135 T3483 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 720137 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 720142 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 720143 T3515 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@156d4b12 name:ZooKeeperConnection Watcher:127.0.0.1:40089/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 720143 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 720145 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 720148 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1
[junit4] 2> 720152 T3509 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 720152 T3515 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 720155 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 720157 T3512 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 720158 T3512 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 720158 T3512 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
[junit4] 2> 720158 T3512 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 720161 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 720162 T3509 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 720163 T3515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 720656 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 720678 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 720694 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 720701 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 720702 T3512 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 720711 T3512 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 720711 T3512 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 720712 T3512 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 720715 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 720818 T3509 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 720818 T3515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 721202 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 721214 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
[junit4] 2> 721220 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721220 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721220 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721223 T3512 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 721224 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721229 T3512 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 721229 T3512 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 721229 T3512 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 721233 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721335 T3509 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 721335 T3515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 721724 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 721736 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
[junit4] 2> 721746 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721746 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721746 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721750 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721750 T3512 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 721755 T3512 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core4",
[junit4] 2> "core_node_name":"node4",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 721756 T3512 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 721756 T3512 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 721759 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 721863 T3509 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 721863 T3515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 722250 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 722263 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722264 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722264 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722266 T3512 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 722267 T3512 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core5",
[junit4] 2> "core_node_name":"node5",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 722267 T3512 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 722267 T3512 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 722272 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722376 T3509 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 722376 T3515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 722763 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
[junit4] 2> 722773 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722773 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722773 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722775 T3512 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 722776 T3512 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core6",
[junit4] 2> "core_node_name":"node6",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 722776 T3512 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 722777 T3512 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
[junit4] 2> 722780 T3511 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 722883 T3509 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 722883 T3515 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 723273 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
[junit4] 2> 723292 T3515 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 723295 T3483 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:40089 40089
[junit4] 2> 723507 T3483 oas.SolrTestCaseJ4.tearDown ###Ending testShardAssignment
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 723511 T3483 oas.SolrTestCaseJ4.setUp ###Starting testDoubleAssignment
[junit4] 2> 723512 T3483 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 723512 T3516 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 723612 T3483 oasc.ZkTestServer.run start zk server on port:35254
[junit4] 2> 723613 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 723617 T3522 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b8fc7e1 name:ZooKeeperConnection Watcher:127.0.0.1:35254/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 723617 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 723618 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 723621 T3524 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f42dae name:ZooKeeperConnection Watcher:127.0.0.1:35254 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 723622 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 723672 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 723674 T3526 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@767ece5f name:ZooKeeperConnection Watcher:127.0.0.1:35254 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 723674 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 723674 T3483 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 723678 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 723682 T3483 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 723685 T3483 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 723688 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 723694 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 723696 T3528 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6fafb7ff name:ZooKeeperConnection Watcher:127.0.0.1:35254/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 723696 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 723698 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 723702 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 723706 T3522 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 723707 T3528 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 723708 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 723709 T3530 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@af80dfd name:ZooKeeperConnection Watcher:127.0.0.1:35254/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 723709 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 723710 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 723713 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 723719 T3483 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:35254_solr
[junit4] 2> 723719 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 723723 T3483 oasc.Overseer.start Overseer (id=91202881827700740-127.0.0.1:35254_solr-n_0000000000) starting
[junit4] 2> 723726 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 723731 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 723735 T3483 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 723739 T3532 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 723741 T3532 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 723741 T3531 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 723745 T3531 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 723745 T3531 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 723745 T3531 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
[junit4] 2> 723746 T3531 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 723749 T3530 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 723752 T3528 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 723752 T3522 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 724243 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 724259 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 724268 T3530 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 724270 T3483 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 724272 T3531 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 724276 T3522 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 724277 T3528 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 724280 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 724280 T3530 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 724281 T3534 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@736e5c name:ZooKeeperConnection Watcher:127.0.0.1:35254/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 724282 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 724284 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 724288 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 724292 T3522 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 724293 T3534 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 724295 T3530 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 724295 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 724297 T3531 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 724297 T3531 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 724301 T3530 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 724306 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 724312 T3530 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 724318 T3530 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 724422 T3534 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 724422 T3522 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 724426 T3483 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 724431 T3522 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 724431 T3534 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 724434 T3483 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:35254 35254
[junit4] 2> 724522 T3483 oas.SolrTestCaseJ4.tearDown ###Ending testDoubleAssignment
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 724527 T3483 oas.SolrTestCaseJ4.setUp ###Starting testReplay
[junit4] 2> 724528 T3483 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 724528 T3535 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 724628 T3483 oasc.ZkTestServer.run start zk server on port:52686
[junit4] 2> 724629 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 724632 T3541 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@135e6929 name:ZooKeeperConnection Watcher:127.0.0.1:52686/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 724632 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 724633 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 724643 T3543 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@787257ea name:ZooKeeperConnection Watcher:127.0.0.1:52686 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 724643 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 724673 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 724674 T3545 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5cc89d0f name:ZooKeeperConnection Watcher:127.0.0.1:52686 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 724674 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 724674 T3483 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 724678 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 724681 T3483 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 724684 T3483 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 724686 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 724691 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 724696 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 724697 T3547 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48e517ff name:ZooKeeperConnection Watcher:127.0.0.1:52686/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 724698 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 724699 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 724702 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 724707 T3483 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:52686_solr
[junit4] 2> 724708 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 724711 T3483 oasc.Overseer.start Overseer (id=91202881894219779-127.0.0.1:52686_solr-n_0000000000) starting
[junit4] 2> 724713 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 724720 T3483 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 724724 T3549 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 724726 T3549 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 724729 T3548 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 724730 T3548 oasc.Overseer$ClusterStateUpdater.run Replaying operations from work queue.
[junit4] 2> 724731 T3548 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "shard":"s1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 724732 T3541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 724737 T3548 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "shard":"s1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core2",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 724738 T3541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 724742 T3548 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 724745 T3548 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 724746 T3548 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "shard":"s1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core3",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 724749 T3547 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 724750 T3541 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 724792 T3515 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 724793 T3515 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard3/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 724793 T3515 oasc.LeaderElector$1.process WARN org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
[junit4] 2> at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
[junit4] 2> at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
[junit4] 2>
[junit4] 2> 724832 T3483 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:52686 52686
[junit4] 2> 724940 T3483 oas.SolrTestCaseJ4.tearDown ###Ending testReplay
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 724944 T3483 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignmentBigger
[junit4] 2> 724944 T3483 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 724945 T3550 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 725045 T3483 oasc.ZkTestServer.run start zk server on port:49707
[junit4] 2> 725046 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725050 T3556 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4835cf4c name:ZooKeeperConnection Watcher:127.0.0.1:49707 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725050 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725079 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725080 T3558 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@672db8cf name:ZooKeeperConnection Watcher:127.0.0.1:49707 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725080 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725080 T3483 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 725084 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725086 T3560 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3db08a20 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725086 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725087 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 725092 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725094 T3562 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@93d7bc3 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725094 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725096 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 725101 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 725110 T3483 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:49707_solr
[junit4] 2> 725110 T3483 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 725116 T3483 oasc.Overseer.start Overseer (id=91202881921417219-127.0.0.1:49707_solr-n_0000000000) starting
[junit4] 2> 725122 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 725130 T3483 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 725135 T3483 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 725138 T3564 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 725139 T3483 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 725141 T3564 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 725141 T3563 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 725145 T3483 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 725147 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725151 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725153 T3566 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73002143 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725153 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725155 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725161 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node0
[junit4] 2> 725166 T3560 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 725167 T3566 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 725168 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725170 T3568 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c0da8b9 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725170 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725172 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725175 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 725179 T3568 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 725180 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725180 T3566 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 725180 T3560 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 725182 T3570 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@474a243c name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725182 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725183 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725187 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
[junit4] 2> 725192 T3568 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 725192 T3560 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 725193 T3566 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 725193 T3570 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 725194 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725195 T3572 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@190cb4a6 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725195 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725197 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725200 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node3
[junit4] 2> 725204 T3570 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 725205 T3572 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 725205 T3566 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 725206 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725206 T3568 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 725206 T3560 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 725207 T3574 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4aa72ff1 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725207 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725209 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725212 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node4
[junit4] 2> 725216 T3572 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 725217 T3566 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 725217 T3560 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 725217 T3574 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 725218 T3570 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 725218 T3568 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 725218 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725219 T3576 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ddc348f name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725220 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725221 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725224 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node5
[junit4] 2> 725229 T3572 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 725230 T3574 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 725230 T3560 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 725230 T3568 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 725231 T3576 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 725231 T3570 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 725232 T3566 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 725232 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725234 T3578 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@497d0b67 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725234 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725236 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725239 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node6
[junit4] 2> 725245 T3568 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725245 T3566 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725245 T3578 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725245 T3574 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725246 T3570 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725246 T3576 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725247 T3560 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725247 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725247 T3572 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 725248 T3580 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@47239608 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725248 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725250 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725253 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node7
[junit4] 2> 725259 T3568 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725259 T3566 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725259 T3578 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725260 T3560 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725260 T3574 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725260 T3576 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725260 T3580 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725261 T3570 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725261 T3572 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 725261 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725262 T3582 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c98e7fb name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725262 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725264 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725267 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node8
[junit4] 2> 725272 T3566 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725274 T3560 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725274 T3580 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725274 T3570 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725274 T3568 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725274 T3576 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725274 T3578 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725275 T3574 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725275 T3572 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725275 T3582 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 725276 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725277 T3584 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78df3975 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725277 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725279 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725282 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node9
[junit4] 2> 725288 T3582 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725290 T3572 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725290 T3560 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725290 T3576 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725290 T3566 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725291 T3580 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725291 T3578 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725291 T3574 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725291 T3568 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725292 T3584 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725292 T3570 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 725293 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725294 T3586 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@40c19611 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725294 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725296 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725299 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node10
[junit4] 2> 725305 T3560 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725307 T3584 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725307 T3566 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725308 T3572 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725308 T3574 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725308 T3580 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725308 T3576 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725308 T3578 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725308 T3586 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725309 T3568 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725310 T3582 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725310 T3570 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 725311 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725311 T3588 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5bb02546 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725311 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725313 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725317 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node11
[junit4] 2> 725324 T3560 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725325 T3584 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725326 T3566 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725326 T3580 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725327 T3568 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725327 T3582 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725327 T3588 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725327 T3570 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725327 T3576 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725327 T3572 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725328 T3574 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725329 T3578 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725329 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725329 T3586 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 725330 T3590 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76174516 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725331 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725332 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725335 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node12
[junit4] 2> 725343 T3584 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725343 T3576 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725344 T3572 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725344 T3578 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725344 T3560 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725344 T3568 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725345 T3574 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725345 T3588 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725345 T3580 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725345 T3570 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725345 T3582 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725346 T3590 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725347 T3566 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725347 T3586 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 725347 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725348 T3592 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a01f813 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725349 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725350 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725353 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node13
[junit4] 2> 725363 T3576 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725363 T3588 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3570 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3582 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3572 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3586 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3580 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3578 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3568 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3574 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3584 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725364 T3560 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725365 T3590 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725366 T3566 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725366 T3592 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 725367 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725368 T3594 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a6d0b59 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725368 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725370 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725373 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node14
[junit4] 2> 725383 T3584 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725383 T3576 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725384 T3566 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725384 T3560 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725384 T3590 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725384 T3572 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725384 T3574 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725384 T3568 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725385 T3578 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725385 T3594 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725385 T3580 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725386 T3588 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725386 T3570 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725386 T3582 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725387 T3592 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725387 T3586 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 725387 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725389 T3596 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c8a404c name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725389 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725391 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725394 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node15
[junit4] 2> 725405 T3584 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725405 T3592 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3580 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3572 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3588 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3590 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3576 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3570 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3560 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725407 T3586 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725407 T3568 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725407 T3574 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725407 T3594 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725406 T3582 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725407 T3566 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725409 T3578 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725409 T3596 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 725410 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725411 T3598 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@278cc34b name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725411 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725412 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725415 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node16
[junit4] 2> 725425 T3584 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725425 T3578 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725426 T3586 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725426 T3580 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725426 T3592 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725426 T3590 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725426 T3570 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725427 T3588 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725427 T3576 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725427 T3572 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725427 T3582 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725427 T3560 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725427 T3596 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725427 T3574 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725428 T3598 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725429 T3594 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725429 T3568 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725429 T3566 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 725429 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725431 T3600 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2261639b name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725431 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725432 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725435 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node17
[junit4] 2> 725446 T3560 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3596 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3594 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3574 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3578 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3580 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3566 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3592 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3568 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725447 T3598 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725448 T3588 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725449 T3576 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725449 T3570 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725449 T3572 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725449 T3600 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725448 T3590 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725449 T3582 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725450 T3584 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725450 T3586 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 725451 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725452 T3602 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49dfc60 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725452 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725453 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725456 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node18
[junit4] 2> 725467 T3600 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725467 T3574 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725467 T3594 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725468 T3578 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725468 T3582 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725468 T3588 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725468 T3592 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725469 T3566 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725469 T3590 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725469 T3586 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725469 T3580 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725469 T3576 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725469 T3570 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725469 T3572 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725470 T3598 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725470 T3596 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725470 T3560 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725471 T3602 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725472 T3584 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725472 T3568 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 725472 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725473 T3604 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10813d3b name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725473 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725475 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725478 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node19
[junit4] 2> 725489 T3600 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725489 T3566 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725490 T3572 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725490 T3596 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725490 T3582 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725490 T3588 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725490 T3584 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725491 T3570 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725491 T3580 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725491 T3576 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725491 T3586 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725491 T3594 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725491 T3602 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725491 T3598 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725492 T3574 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725492 T3560 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725492 T3568 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725492 T3578 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725492 T3604 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725494 T3592 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725494 T3483 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 725494 T3590 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 725495 T3606 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c69d124 name:ZooKeeperConnection Watcher:127.0.0.1:49707/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 725496 T3483 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 725497 T3483 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 725500 T3483 oascc.SolrZkClient.makePath makePath: /live_nodes/node20
[junit4] 2> 725511 T3600 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 725511 T3566 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 725512 T3594 oascc.ZkStateReader$3.process Updating live nodes... (21)
[...truncated too long message...]
ailure.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
[junit4] 2> at java.lang.Thread.run(Thread.java:662)
[junit4] 2>
[junit4] 2> 745769 T3898 oasc.SolrException.log ERROR Problem cleaning up collection in zk:collection1:org.apache.zookeeper.KeeperException$NotEmptyException: KeeperErrorCode = Directory not empty for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:125)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:173)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:170)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:170)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:654)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.removeCore(Overseer.java:875)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.processMessage(Overseer.java:226)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:187)
[junit4] 2> at java.lang.Thread.run(Thread.java:662)
[junit4] 2>
[junit4] 2> 745770 T3891 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 745773 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 745778 T3897 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 745784 T3897 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 745785 T3885 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 745804 T3880 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1440489f9120000, likely client has closed socket
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
[junit4] 2> at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
[junit4] 2> at java.lang.Thread.run(Thread.java:662)
[junit4] 2>
[junit4] 2> 745805 T3483 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:38491 38491
[junit4] 2> 745877 T3483 oas.SolrTestCaseJ4.tearDown ###Ending testOverseerFailure
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OverseerTest -Dtests.method=testOverseerFailure -Dtests.seed=3A671B75C53DB535 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_PR -Dtests.timezone=Pacific/Tongatapu -Dtests.file.encoding=ISO-8859-1
[junit4] ERROR 3.12s J1 | OverseerTest.testOverseerFailure <<<
[junit4] > Throwable #1: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
[junit4] > at __randomizedtesting.SeedInfo.seed([3A671B75C53DB535:3E6F9486D7985A14]:0)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:428)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:425)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
[junit4] > at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
[junit4] > at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
[junit4] > at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
[junit4] > at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
[junit4] > at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
[junit4] > at java.lang.Thread.run(Thread.java:662)
[junit4] 2> 745887 T3483 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> 745887 T3483 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 747286 T3891 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
[junit4] 2> 748887 T3483 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4] 2> NOTE: test params are: codec=Lucene45, sim=DefaultSimilarity, locale=es_PR, timezone=Pacific/Tongatapu
[junit4] 2> NOTE: Linux 3.8.0-35-generic amd64/Sun Microsystems Inc. 1.6.0_45 (64-bit)/cpus=8,threads=1,free=111644352,total=354811904
[junit4] 2> NOTE: All tests run in this JVM: [BasicDistributedZk2Test, ChangedSchemaMergeTest, SolrCoreCheckLockOnStartupTest, TestSerializedLuceneMatchVersion, PluginInfoTest, DocValuesMissingTest, TestPhraseSuggestions, TestCoreDiscovery, HdfsCollectionsAPIDistributedZkTest, TestAtomicUpdateErrorCases, FieldAnalysisRequestHandlerTest, ChaosMonkeySafeLeaderTest, DisMaxRequestHandlerTest, SolrTestCaseJ4Test, PrimUtilsTest, TestReload, TestRangeQuery, CustomCollectionTest, NotRequiredUniqueKeyTest, AddSchemaFieldsUpdateProcessorFactoryTest, FullSolrCloudDistribCmdsTest, ZkSolrClientTest, HdfsUnloadDistributedZkTest, SuggesterTSTTest, TestFastWriter, TestQuerySenderNoQuery, TestDFRSimilarityFactory, TestHashPartitioner, TestMultiCoreConfBootstrap, SolrCmdDistributorTest, TestStressLucene, StressHdfsTest, SolrRequestParserTest, TestSolrQueryParserDefaultOperatorResource, ZkNodePropsTest, TestQuerySenderListener, TestDistributedSearch, HdfsDirectoryTest, SpellCheckCollatorTest, DocValuesMultiTest, TestOmitPositions, TestSolr4Spatial, TestSchemaNameResource, TestBinaryResponseWriter, TestShardHandlerFactory, TestStressReorder, JSONWriterTest, HighlighterTest, SolrIndexConfigTest, TestSolrJ, TestCopyFieldCollectionResource, SuggesterWFSTTest, SimplePostToolTest, TestManagedSchemaFieldResource, TestLazyCores, TestIndexSearcher, TestSimpleQParserPlugin, BadIndexSchemaTest, TestSolrDeletionPolicy2, TestFieldTypeResource, AssignTest, TestImplicitCoreProperties, TestStressRecovery, PreAnalyzedFieldTest, TestIBSimilarityFactory, TestQueryTypes, DirectUpdateHandlerTest, HdfsLockFactoryTest, TestRandomFaceting, CurrencyFieldXmlFileTest, TestIndexingPerformance, TestFastOutputStream, PathHierarchyTokenizerFactoryTest, TestCollationField, TestFileDictionaryLookup, DateFieldTest, SpellingQueryConverterTest, SortByFunctionTest, StandardRequestHandlerTest, PingRequestHandlerTest, CurrencyFieldOpenExchangeTest, StatsComponentTest, CoreContainerCoreInitFailuresTest, IndexSchemaRuntimeFieldTest, DistributedSuggestComponentTest, TestRTGBase, BasicDistributedZkTest, UnloadDistributedZkTest, OpenCloseCoreStressTest, OverseerTest]
[junit4] Completed on J1 in 29.58s, 8 tests, 1 error <<< FAILURES!
[...truncated 541 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:459: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:439: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:491: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1308: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:941: There were test failures: 358 suites, 1552 tests, 1 error, 35 ignored (7 assumptions)
Total time: 45 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.6.0_45 -XX:+UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure