You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/03/21 01:24:39 UTC
[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1944 - Failure
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1944/
1 tests failed.
REGRESSION: org.apache.solr.cloud.OverseerTest.testOverseerFailure
Error Message:
Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
Stack Trace:
org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
at __randomizedtesting.SeedInfo.seed([1E9BB6D73A6773C3:1A93392428C29CE2]:0)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:135)
at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:155)
at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:666)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1617)
at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:826)
at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:862)
at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:876)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:783)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:443)
at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:835)
at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:737)
at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:771)
at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:782)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:122)
... 45 more
Caused by: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:431)
at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:428)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:385)
at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:372)
at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:127)
at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)
... 46 more
Build Log:
[...truncated 10680 lines...]
[junit4] Suite: org.apache.solr.cloud.OverseerTest
[junit4] 2> 1917319 T9156 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
[junit4] 2> 1917319 T9156 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J1/./solrtest-OverseerTest-1395361045246
[junit4] 2> 1917320 T9156 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 1917325 T9156 oas.SolrTestCaseJ4.setUp ###Starting testPlaceholders
[junit4] 2> 1917325 T9156 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 1917326 T9157 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1917427 T9156 oasc.ZkTestServer.run start zk server on port:62785
[junit4] 2> 1917428 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1917432 T9163 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b4b2470 name:ZooKeeperConnection Watcher:127.0.0.1:62785/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1917432 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1917433 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1917434 T9165 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c0c5f name:ZooKeeperConnection Watcher:127.0.0.1:62785 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1917435 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1917437 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1917447 T9167 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@328d56a8 name:ZooKeeperConnection Watcher:127.0.0.1:62785 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1917447 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1917447 T9156 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1917450 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1917459 T9156 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1917461 T9156 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1917462 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1917465 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1917467 T9169 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cdf1736 name:ZooKeeperConnection Watcher:127.0.0.1:62785/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1917467 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1917468 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1917470 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 1917472 T9169 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 1917472 T9163 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 1917473 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1917474 T9171 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b3e08ae name:ZooKeeperConnection Watcher:127.0.0.1:62785/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1917474 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1917475 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1917477 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1917481 T9156 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62785_solr
[junit4] 2> 1917481 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1917483 T9156 oasc.Overseer.start Overseer (id=91446381461962756-127.0.0.1:62785_solr-n_0000000000) starting
[junit4] 2> 1917485 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1917488 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1917491 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1917494 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1917496 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1917499 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1917514 T9173 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1917515 T9173 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 1917515 T9172 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1917518 T9172 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1917518 T9172 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"12",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1917519 T9172 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
[junit4] 2> 1917519 T9172 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard7
[junit4] 2> 1917530 T9171 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1917531 T9163 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1917531 T9169 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1918018 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard7/election
[junit4] 2> 1918196 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard7
[junit4] 2> 1918201 T9171 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1918202 T9156 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1918202 T9172 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1918204 T9169 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 1918204 T9163 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 1918207 T9156 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62785 62785
[junit4] 2> 1918478 T9156 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1918484 T9156 oas.SolrTestCaseJ4.setUp ###Starting testOverseerFailure
[junit4] 2> 1918485 T9156 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 1918486 T9174 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1918586 T9156 oasc.ZkTestServer.run start zk server on port:62795
[junit4] 2> 1918586 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1918590 T9180 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@684c1801 name:ZooKeeperConnection Watcher:127.0.0.1:62795 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1918591 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1918636 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1918638 T9182 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@53bd2000 name:ZooKeeperConnection Watcher:127.0.0.1:62795 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1918638 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1918638 T9156 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1918641 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1918642 T9184 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@149ea865 name:ZooKeeperConnection Watcher:127.0.0.1:62795/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1918642 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1918642 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1918644 T9156 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1918646 T9156 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1918647 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1918649 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1918650 T9186 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@47c65448 name:ZooKeeperConnection Watcher:127.0.0.1:62795/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1918650 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1918651 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1918652 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 1918654 T9184 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 1918655 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1918654 T9186 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 1918656 T9188 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58c2c953 name:ZooKeeperConnection Watcher:127.0.0.1:62795/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1918656 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1918657 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1918658 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1918661 T9156 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62795_solr
[junit4] 2> 1918661 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1918663 T9156 oasc.Overseer.start Overseer (id=91446381537853444-127.0.0.1:62795_solr-n_0000000000) starting
[junit4] 2> 1918664 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1918666 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1918668 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1918670 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1918672 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1918676 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1918690 T9190 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1918690 T9190 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 1918690 T9189 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1919693 T9188 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1919694 T9189 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1919695 T9189 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1919695 T9189 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
[junit4] 2> 1919695 T9189 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1919710 T9188 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1919710 T9186 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1919710 T9184 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1920194 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1920211 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1920216 T9188 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1920216 T9156 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1920217 T9189 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1920219 T9188 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1920219 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1920220 T9189 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1920220 T9189 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1920223 T9188 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1920325 T9184 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1920325 T9186 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1921380 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1921398 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1921399 T9192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@344e5210 name:ZooKeeperConnection Watcher:127.0.0.1:62795/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1921400 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1921404 T9156 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62795_solr
[junit4] 2> 1921404 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1921406 T9156 oasc.Overseer.start Overseer (id=91446381537853445-127.0.0.1:62795_solr-n_0000000001) starting
[junit4] 2> 1921416 T9194 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1921416 T9194 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 1921416 T9193 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1921418 T9193 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1921418 T9193 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"recovering",
[junit4] 2> "node_name":"node1",
[junit4] 2> "core":"core1",
[junit4] 2> "core_node_name":"core_node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "numShards":"1",
[junit4] 2> "base_url":"http://node1/solr/"}
[junit4] 2> 1921418 T9193 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1921421 T9192 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1921431 T9184 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1921431 T9186 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1921484 T9192 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1921484 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
[junit4] 2> 1921485 T9193 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1921500 T9186 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1921500 T9156 oasc.LeaderElector.checkIfIamLeader WARN Failed setting watch org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election/91446381537853443-node1_core1-n_0000000002
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:276)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:273)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:273)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:125)
[junit4] 2> at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
[junit4] 2> at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:155)
[junit4] 2> at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:666)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[junit4] 2> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
[junit4] 2> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[junit4] 2> at java.lang.reflect.Method.invoke(Method.java:606)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1617)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:826)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:862)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:876)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
[junit4] 2> at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:783)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:443)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:835)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:737)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:771)
[junit4] 2> at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:782)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
[junit4] 2> at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
[junit4] 2> at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
[junit4] 2> at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
[junit4] 2> at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
[junit4] 2> at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
[junit4] 2> at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
[junit4] 2> at java.lang.Thread.run(Thread.java:724)
[junit4] 2>
[junit4] 2> 1921508 T9193 oasc.SolrException.log ERROR Problem cleaning up collection in zk:collection1:org.apache.zookeeper.KeeperException$NotEmptyException: KeeperErrorCode = Directory not empty for /collections/collection1/leader_elect/shard1/election
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:125)
[junit4] 2> at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] 2> at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:176)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:173)
[junit4] 2> at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:173)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:635)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:630)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:630)
[junit4] 2> at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:630)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.removeCore(Overseer.java:986)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.processMessage(Overseer.java:263)
[junit4] 2> at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:212)
[junit4] 2> at java.lang.Thread.run(Thread.java:724)
[junit4] 2>
[junit4] 2> 1921508 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1921511 T9192 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1921512 T9192 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1921514 T9186 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1921514 T9193 oasc.Overseer$ClusterStateUpdater.setShardLeader ERROR Could not mark shard leader for non existing collection:collection1
[junit4] 2> 1921527 T9192 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1921529 T9192 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1921530 T9193 oasc.Overseer$ClusterStateUpdater.setShardLeader ERROR Could not mark shard leader for non existing collection:collection1
[junit4] 2> 1921532 T9192 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1921634 T9184 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1921634 T9186 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1922513 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1923516 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1924525 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1925529 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1926532 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1927535 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1928539 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1929543 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1930573 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1931576 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1932587 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1933591 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1934599 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1935629 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1936638 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
[junit4] 2> 1936642 T9186 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 1936653 T9184 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 1936657 T9156 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62795 62795
[junit4] 2> 1936787 T9156 oas.SolrTestCaseJ4.tearDown ###Ending testOverseerFailure
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> NOTE: reproduce with: ant test -Dtestcase=OverseerTest -Dtests.method=testOverseerFailure -Dtests.seed=1E9BB6D73A6773C3 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=el_CY -Dtests.timezone=America/Indiana/Winamac -Dtests.file.encoding=US-ASCII
[junit4] ERROR 18.3s J1 | OverseerTest.testOverseerFailure <<<
[junit4] > Throwable #1: org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
[junit4] > at __randomizedtesting.SeedInfo.seed([1E9BB6D73A6773C3:1A93392428C29CE2]:0)
[junit4] > at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:135)
[junit4] > at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
[junit4] > at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
[junit4] > at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
[junit4] > at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:155)
[junit4] > at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:666)
[junit4] > at java.lang.Thread.run(Thread.java:724)
[junit4] > Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
[junit4] > at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
[junit4] > at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:122)
[junit4] > ... 45 more
[junit4] > Caused by: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
[junit4] > at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
[junit4] > at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:431)
[junit4] > at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:428)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:385)
[junit4] > at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:372)
[junit4] > at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:127)
[junit4] > at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)
[junit4] > ... 46 more
[junit4] 2> 1936824 T9156 oas.SolrTestCaseJ4.setUp ###Starting testStateChange
[junit4] 2> 1936824 T9156 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 1936825 T9195 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1936926 T9156 oasc.ZkTestServer.run start zk server on port:62846
[junit4] 2> 1936926 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1936930 T9201 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@164f6fe2 name:ZooKeeperConnection Watcher:127.0.0.1:62846/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1936930 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1936931 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1936933 T9203 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a7bb33 name:ZooKeeperConnection Watcher:127.0.0.1:62846 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1936934 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1936982 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1936983 T9205 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f0efde5 name:ZooKeeperConnection Watcher:127.0.0.1:62846 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1936983 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1936984 T9156 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1936986 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1936988 T9156 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1936989 T9156 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1936991 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1936993 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1936994 T9207 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f0ee336 name:ZooKeeperConnection Watcher:127.0.0.1:62846/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1936994 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1936995 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1936997 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1937000 T9156 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62846_solr
[junit4] 2> 1937001 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1937002 T9156 oasc.Overseer.start Overseer (id=91446382740242435-127.0.0.1:62846_solr-n_0000000000) starting
[junit4] 2> 1937004 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1937007 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1937009 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1937012 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1937014 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1937017 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1937022 T9209 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1937022 T9208 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1937022 T9209 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 1937024 T9156 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1937024 T9208 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1937025 T9208 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"recovering"}
[junit4] 2> 1937025 T9208 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1937027 T9207 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1937028 T9201 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 1937075 T9156 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1937077 T9207 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1937078 T9208 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1937078 T9208 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
[junit4] 2> "operation":"state",
[junit4] 2> "base_url":"http://127.0.0.1/solr",
[junit4] 2> "node_name":"node1",
[junit4] 2> "collection":"collection1",
[junit4] 2> "core":"core1",
[junit4] 2> "roles":"",
[junit4] 2> "state":"active"}
[junit4] 2> 1937078 T9208 oasc.Overseer$ClusterStateUpdater.updateState node=core_node1 is already registered
[junit4] 2> 1937079 T9208 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
[junit4] 2> 1937089 T9207 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1937191 T9201 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
[junit4] 2> 1937232 T9156 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:62846 62846
[junit4] 2> 1937296 T9156 oas.SolrTestCaseJ4.tearDown ###Ending testStateChange
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1937303 T9156 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignmentBigger
[junit4] 2> 1937303 T9156 oasc.ZkTestServer.run STARTING ZK TEST SERVER
[junit4] 2> 1937304 T9210 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
[junit4] 2> 1937405 T9156 oasc.ZkTestServer.run start zk server on port:62851
[junit4] 2> 1937406 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937409 T9216 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15e57761 name:ZooKeeperConnection Watcher:127.0.0.1:62851 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937409 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937438 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937440 T9218 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66902a59 name:ZooKeeperConnection Watcher:127.0.0.1:62851 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937440 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937440 T9156 oascc.SolrZkClient.makePath makePath: /solr
[junit4] 2> 1937443 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937444 T9220 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@61c487af name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937444 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937445 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes
[junit4] 2> 1937447 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937448 T9222 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55adc308 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937448 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937449 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect
[junit4] 2> 1937451 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
[junit4] 2> 1937455 T9156 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62851_solr
[junit4] 2> 1937455 T9156 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
[junit4] 2> 1937457 T9156 oasc.Overseer.start Overseer (id=91446382771765251-127.0.0.1:62851_solr-n_0000000000) starting
[junit4] 2> 1937459 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue
[junit4] 2> 1937461 T9156 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
[junit4] 2> 1937464 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
[junit4] 2> 1937467 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
[junit4] 2> 1937469 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
[junit4] 2> 1937472 T9156 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
[junit4] 2> 1937477 T9224 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
[junit4] 2> 1937477 T9156 oascc.SolrZkClient.makePath makePath: /clusterstate.json
[junit4] 2> 1937478 T9223 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
[junit4] 2> 1937478 T9224 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
[junit4] 2> 1937479 T9156 oascc.SolrZkClient.makePath makePath: /aliases.json
[junit4] 2> 1937481 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937483 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937484 T9226 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e4e9bf9 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937484 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937485 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937487 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node0
[junit4] 2> 1937489 T9226 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 1937489 T9220 oascc.ZkStateReader$3.process Updating live nodes... (1)
[junit4] 2> 1937489 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937491 T9228 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1494c28a name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937491 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937492 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937493 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
[junit4] 2> 1937495 T9228 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 1937496 T9226 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 1937496 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937496 T9220 oascc.ZkStateReader$3.process Updating live nodes... (2)
[junit4] 2> 1937497 T9230 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@258e5464 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937498 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937499 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937500 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
[junit4] 2> 1937502 T9226 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1937502 T9220 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1937502 T9230 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1937503 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937502 T9228 oascc.ZkStateReader$3.process Updating live nodes... (3)
[junit4] 2> 1937504 T9232 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@349b01f1 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937504 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937505 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937507 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node3
[junit4] 2> 1937509 T9230 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1937509 T9226 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1937509 T9220 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1937509 T9232 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1937509 T9228 oascc.ZkStateReader$3.process Updating live nodes... (4)
[junit4] 2> 1937510 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937511 T9234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@159c864c name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937511 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937512 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937514 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node4
[junit4] 2> 1937517 T9232 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 1937517 T9226 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 1937517 T9228 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 1937517 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937517 T9230 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 1937517 T9220 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 1937517 T9234 oascc.ZkStateReader$3.process Updating live nodes... (5)
[junit4] 2> 1937519 T9236 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68358d28 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937519 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937520 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937521 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node5
[junit4] 2> 1937524 T9226 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 1937524 T9234 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 1937524 T9228 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 1937524 T9232 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 1937524 T9230 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 1937524 T9236 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 1937524 T9220 oascc.ZkStateReader$3.process Updating live nodes... (6)
[junit4] 2> 1937524 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937531 T9238 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3df7b46c name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937531 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937532 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937534 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node6
[junit4] 2> 1937544 T9230 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937544 T9228 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937545 T9232 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937544 T9236 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937544 T9220 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937544 T9226 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937544 T9238 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937544 T9234 oascc.ZkStateReader$3.process Updating live nodes... (7)
[junit4] 2> 1937545 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937547 T9240 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3cb1be41 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937547 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937548 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937550 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node7
[junit4] 2> 1937552 T9236 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937552 T9226 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937552 T9230 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937552 T9234 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937553 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937552 T9238 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937552 T9228 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937552 T9232 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937552 T9220 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937552 T9240 oascc.ZkStateReader$3.process Updating live nodes... (8)
[junit4] 2> 1937554 T9242 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66541c name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937555 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937556 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937557 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node8
[junit4] 2> 1937560 T9236 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9228 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9242 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9230 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9234 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9220 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9238 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9226 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9232 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9240 oascc.ZkStateReader$3.process Updating live nodes... (9)
[junit4] 2> 1937560 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937562 T9244 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57907804 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937562 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937563 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937565 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node9
[junit4] 2> 1937567 T9244 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937567 T9228 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937568 T9240 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937568 T9238 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937568 T9226 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937568 T9234 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937568 T9242 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937567 T9232 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937567 T9230 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937567 T9236 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937567 T9220 oascc.ZkStateReader$3.process Updating live nodes... (10)
[junit4] 2> 1937568 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937570 T9246 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25b2e33f name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937570 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937571 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937573 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node10
[junit4] 2> 1937575 T9238 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937575 T9228 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937575 T9244 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937576 T9234 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937576 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937575 T9220 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937576 T9246 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937576 T9242 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937575 T9232 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937575 T9230 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937578 T9248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17dd2efc name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937575 T9240 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937575 T9236 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937575 T9226 oascc.ZkStateReader$3.process Updating live nodes... (11)
[junit4] 2> 1937578 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937580 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937581 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node11
[junit4] 2> 1937584 T9248 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9232 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9242 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9244 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9228 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9246 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9234 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9220 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9226 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9238 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9240 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937584 T9236 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937586 T9250 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@644367dd name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937584 T9230 oascc.ZkStateReader$3.process Updating live nodes... (12)
[junit4] 2> 1937585 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937587 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937588 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937590 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node12
[junit4] 2> 1937592 T9234 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9246 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9226 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937593 T9248 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937593 T9232 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937593 T9242 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9238 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9230 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9228 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9244 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9250 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9220 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9236 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937592 T9240 oascc.ZkStateReader$3.process Updating live nodes... (13)
[junit4] 2> 1937594 T9252 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2f22c2fd name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937593 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937596 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937597 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937598 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node13
[junit4] 2> 1937601 T9246 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9236 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9220 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9252 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9226 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937602 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937601 T9238 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9234 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9242 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9232 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9228 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9244 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9230 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9248 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9250 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937601 T9240 oascc.ZkStateReader$3.process Updating live nodes... (14)
[junit4] 2> 1937603 T9254 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@23f9f545 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937605 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937606 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937607 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node14
[junit4] 2> 1937610 T9254 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9248 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9242 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9230 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937610 T9244 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9228 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9250 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9232 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9246 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9234 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9238 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9240 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9252 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9226 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9236 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937610 T9220 oascc.ZkStateReader$3.process Updating live nodes... (15)
[junit4] 2> 1937612 T9256 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60a036f9 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937614 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937615 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937617 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node15
[junit4] 2> 1937619 T9254 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937619 T9246 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9244 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9238 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9248 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9242 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937619 T9256 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9232 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9240 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9226 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9228 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937620 T9252 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937619 T9220 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937619 T9234 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937619 T9236 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937619 T9250 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937619 T9230 oascc.ZkStateReader$3.process Updating live nodes... (16)
[junit4] 2> 1937622 T9258 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28080b7d name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937621 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937624 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937625 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937626 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node16
[junit4] 2> 1937629 T9258 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9240 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9246 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9226 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9244 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9228 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9252 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9220 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9236 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9254 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9238 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9232 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9250 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9256 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9248 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9242 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9234 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937629 T9230 oascc.ZkStateReader$3.process Updating live nodes... (17)
[junit4] 2> 1937631 T9260 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a5c401b name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937630 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937633 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937634 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937636 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node17
[junit4] 2> 1937639 T9248 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9238 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9258 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9250 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9252 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9260 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9244 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9254 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9236 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9220 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9240 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9242 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9226 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9230 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9228 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937639 T9232 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937659 T9262 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b9227df name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937657 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937657 T9246 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937657 T9234 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937657 T9256 oascc.ZkStateReader$3.process Updating live nodes... (18)
[junit4] 2> 1937661 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937663 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937664 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node18
[junit4] 2> 1937667 T9232 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937667 T9248 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937667 T9244 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9254 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9256 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9220 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9234 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937669 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937668 T9246 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9236 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9230 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9262 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9252 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937670 T9264 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69ad9456 name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937668 T9260 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9238 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9240 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937668 T9250 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937667 T9228 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937667 T9226 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937667 T9242 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937667 T9258 oascc.ZkStateReader$3.process Updating live nodes... (19)
[junit4] 2> 1937670 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937673 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937675 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node19
[junit4] 2> 1937677 T9232 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9220 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9252 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9240 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9262 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937679 T9244 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9234 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937679 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937678 T9246 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9256 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9236 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9226 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9254 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9258 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9228 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9260 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9242 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9250 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9238 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937678 T9230 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937677 T9248 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937677 T9264 oascc.ZkStateReader$3.process Updating live nodes... (20)
[junit4] 2> 1937680 T9266 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f4b0d3a name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937683 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937684 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937685 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node20
[junit4] 2> 1937688 T9248 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9226 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9242 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9236 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937689 T9228 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937689 T9266 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937689 T9244 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937689 T9156 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
[junit4] 2> 1937689 T9220 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937689 T9234 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937689 T9260 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9230 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937689 T9256 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937691 T9268 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@580bdeea name:ZooKeeperConnection Watcher:127.0.0.1:62851/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
[junit4] 2> 1937688 T9246 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9264 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9262 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9254 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9252 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9240 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9232 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9258 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9250 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937688 T9238 oascc.ZkStateReader$3.process Updating live nodes... (21)
[junit4] 2> 1937691 T9156 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
[junit4] 2> 1937694 T9156 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper...
[junit4] 2> 1937696 T9156 oascc.SolrZkClient.makePath makePath: /live_nodes/node21
[junit4] 2> 1937699 T9266 oascc.ZkStateReader$3.process Updating live nodes... (22)
[junit4] 2> 1937699 T9220 oascc.ZkStateReader$3.process Updating live nodes... (22)
[junit4] 2> 1937699 T9254 oascc.ZkStateReader$3.process Updating live nodes... (22)
[junit4] 2> 1937699 T9240 oascc.ZkStateReader$3.process Updating live nodes... (22)
[junit4] 2> 1937700 T9258 oascc.ZkStateReader$3.process Updating live nodes... (22)
[junit4] 2> 1937700 T9226 oascc.ZkStateReader$3.process Updating live nodes... (22)
[junit4] 2> 1937700 T9238 oascc.ZkStateReader$3.process Updating live nodes... (22)
[junit4] 2> 1937700
[...truncated too long message...]
ildrenChanged
[junit4] 2> 1959477 T9714 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1959481 T9714 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core2",
[junit4] 2> "core_node_name":"node2",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1959481 T9714 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1959481 T9714 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
[junit4] 2> 1959484 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1959589 T9711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1959589 T9717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1959978 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection2/leader_elect/shard2/election
[junit4] 2> 1959996 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection2/leaders/shard2
[junit4] 2> 1960000 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960000 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960000 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960002 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960002 T9714 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1960006 T9714 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
[junit4] 2> "operation":"state",
[junit4] 2> "state":"active",
[junit4] 2> "node_name":"127.0.0.1",
[junit4] 2> "core":"core3",
[junit4] 2> "core_node_name":"node3",
[junit4] 2> "collection":"collection2",
[junit4] 2> "numShards":"3",
[junit4] 2> "base_url":"http://127.0.0.1/solr/"}
[junit4] 2> 1960006 T9714 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
[junit4] 2> 1960006 T9714 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
[junit4] 2> 1960016 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960119 T9717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1960119 T9711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1960503 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection2/leader_elect/shard1/election
[junit4] 2> 1960518 T9156 oascc.SolrZkClient.makePath makePath: /collections/collection2/leaders/shard1
[junit4] 2> 1960522 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960522 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960523 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960523 T9714 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper...
[junit4] 2> 1960526 T9713 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
[junit4] 2> 1960628 T9711 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1960628 T9717 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
[junit4] 2> 1960676 T9717 oascc.ZkStateReader$3.process Updating live nodes... (0)
[junit4] 2> 1960679 T9156 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:63051 63051
[junit4] 2> 1960736 T9156 oas.SolrTestCaseJ4.tearDown ###Ending testBadQueueItem
[junit4] 1> client port:0.0.0.0/0.0.0.0:0
[junit4] 2> 1960738 T9156 oas.SolrTestCaseJ4.initCore ####initCore
[junit4] 2> 1960738 T9156 oas.SolrTestCaseJ4.initCore ####initCore end
[junit4] 2> 1963740 T9156 oas.SolrTestCaseJ4.deleteCore ###deleteCore
[junit4] 2> NOTE: test params are: codec=Appending, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=el_CY, timezone=America/Indiana/Winamac
[junit4] 2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=142147024,total=223346688
[junit4] 2> NOTE: All tests run in this JVM: [TestSchemaResource, TestSort, SliceStateTest, CircularListTest, BlockDirectoryTest, TestPhraseSuggestions, TestDFRSimilarityFactory, UnloadDistributedZkTest, TestDefaultSearchFieldResource, HdfsLockFactoryTest, ShardRoutingTest, SolrCoreCheckLockOnStartupTest, TestValueSourceCache, XmlUpdateRequestHandlerTest, TestHighlightDedupGrouping, DistribCursorPagingTest, TestPostingsSolrHighlighter, TestLMDirichletSimilarityFactory, DistributedQueryElevationComponentTest, NotRequiredUniqueKeyTest, TestLazyCores, TriLevelCompositeIdRoutingTest, RequiredFieldsTest, TestLFUCache, DistributedQueryComponentCustomSortTest, BasicDistributedZk2Test, WordBreakSolrSpellCheckerTest, TestSurroundQueryParser, TestSolrDeletionPolicy2, CSVRequestHandlerTest, UpdateParamsTest, TestConfig, TestSolrJ, AssignTest, BadIndexSchemaTest, HdfsChaosMonkeySafeLeaderTest, TestLMJelinekMercerSimilarityFactory, TestMultiCoreConfBootstrap, TestSearcherReuse, TestBlendedInfixSuggestions, DeleteReplicaTest, EchoParamsTest, CollectionsAPIDistributedZkTest, ZkCLITest, TestBinaryResponseWriter, TestHighFrequencyDictionaryFactory, DistributedExpandComponentTest, SuggestComponentTest, DocValuesTest, TestAtomicUpdateErrorCases, TestOmitPositions, ClusterStateTest, TestShardHandlerFactory, CoreAdminHandlerTest, StressHdfsTest, HdfsRecoveryZkTest, TestFoldingMultitermQuery, TestStressRecovery, ChaosMonkeySafeLeaderTest, TestDynamicFieldResource, TestDynamicFieldCollectionResource, TestSchemaSimilarityResource, CachingDirectoryFactoryTest, ResourceLoaderTest, StandardRequestHandlerTest, ZkNodePropsTest, TestNumberUtils, TestDefaultSimilarityFactory, TestWriterPerf, TermVectorComponentDistributedTest, URLClassifyProcessorTest, TestSimpleQParserPlugin, ShowFileRequestHandlerTest, TestSolrXMLSerializer, DocValuesMissingTest, DOMUtilTest, BadComponentTest, SpellCheckCollatorTest, QueryElevationComponentTest, OpenExchangeRatesOrgProviderTest, ShardSplitTest, CoreContainerCoreInitFailuresTest, ConvertedLegacyTest, LeaderElectionTest, AsyncMigrateRouteKeyTest, RequestHandlersTest, TestFaceting, TestReload, TestCSVResponseWriter, ConnectionManagerTest, TestIndexSearcher, DirectSolrSpellCheckerTest, LegacyHTMLStripCharFilterTest, BasicDistributedZkTest, HdfsSyncSliceTest, SSLMigrationTest, TestCloudManagedSchema, LeaderElectionIntegrationTest, DeleteInactiveReplicaTest, RecoveryZkTest, OverseerTest]
[junit4] Completed on J1 in 46.48s, 9 tests, 1 error <<< FAILURES!
[...truncated 540 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:490: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1276: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:908: There were test failures: 377 suites, 1597 tests, 1 error, 37 ignored (16 assumptions)
Total time: 66 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure