You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/03/18 10:39:46 UTC

[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1938 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1938/

1 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerTest.testOverseerFailure

Error Message:
Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed

Stack Trace:
org.apache.solr.common.SolrException: Could not register as the leader because creating the ephemeral registration node in ZooKeeper failed
	at __randomizedtesting.SeedInfo.seed([D7C8C96EC9E09331:D3C0469DDB457C10]:0)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:135)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:156)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:289)
	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:155)
	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:666)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1617)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:826)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:862)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:876)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:783)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:443)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:835)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:771)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:782)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:359)
	at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:122)
	... 45 more
Caused by: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:431)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:428)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:385)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:372)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:127)
	at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)
	... 46 more




Build Log:
[...truncated 10194 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerTest
   [junit4]   2> 271753 T2414 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 271753 T2414 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-OverseerTest-1395132935001
   [junit4]   2> 271754 T2414 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 271758 T2414 oas.SolrTestCaseJ4.setUp ###Starting testStateChange
   [junit4]   2> 271759 T2414 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 271759 T2415 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 271860 T2414 oasc.ZkTestServer.run start zk server on port:26669
   [junit4]   2> 271861 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271865 T2421 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f95dccc name:ZooKeeperConnection Watcher:127.0.0.1:26669/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271865 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271866 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271868 T2423 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@59fed135 name:ZooKeeperConnection Watcher:127.0.0.1:26669 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271868 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271871 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271879 T2425 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d951ea3 name:ZooKeeperConnection Watcher:127.0.0.1:26669 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271879 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271879 T2414 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 271883 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 271886 T2414 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 271895 T2414 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 271897 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 271900 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 271902 T2427 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41bf883d name:ZooKeeperConnection Watcher:127.0.0.1:26669/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 271902 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 271903 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 271906 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 271918 T2414 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:26669_solr
   [junit4]   2> 271919 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 271921 T2414 oasc.Overseer.start Overseer (id=91431432029077507-127.0.0.1:26669_solr-n_0000000000) starting
   [junit4]   2> 271931 T2414 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 271934 T2414 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 271938 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 271941 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 271944 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 271948 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 271954 T2429 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 271954 T2429 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 271954 T2428 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 271956 T2414 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 271957 T2428 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 271957 T2428 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "roles":"",
   [junit4]   2> 	  "state":"recovering"}
   [junit4]   2> 271957 T2428 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 271960 T2427 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 271961 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 272008 T2414 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 272011 T2427 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 272012 T2428 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 272013 T2428 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=null message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "roles":"",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 272013 T2428 oasc.Overseer$ClusterStateUpdater.updateState node=core_node1 is already registered
   [junit4]   2> 272013 T2428 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 272016 T2427 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 272118 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 272167 T2414 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26669 26669
   [junit4]   2> 272270 T2414 oas.SolrTestCaseJ4.tearDown ###Ending testStateChange
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 272278 T2414 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignmentBigger
   [junit4]   2> 272279 T2414 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 272279 T2430 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 272380 T2414 oasc.ZkTestServer.run start zk server on port:26676
   [junit4]   2> 272380 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272384 T2436 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69cd3e06 name:ZooKeeperConnection Watcher:127.0.0.1:26676 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272384 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272440 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272442 T2438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@111e4d72 name:ZooKeeperConnection Watcher:127.0.0.1:26676 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272442 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272443 T2414 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 272447 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272455 T2440 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7db06608 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272455 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272455 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 272458 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272460 T2442 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6cfdd53f name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272460 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272462 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 272464 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 272469 T2414 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:26676_solr
   [junit4]   2> 272469 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 272471 T2414 oasc.Overseer.start Overseer (id=91431432063287299-127.0.0.1:26676_solr-n_0000000000) starting
   [junit4]   2> 272474 T2414 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 272477 T2414 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 272481 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 272484 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 272488 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 272492 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 272498 T2444 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 272498 T2414 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 272498 T2444 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 272499 T2443 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 272500 T2414 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 272502 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272506 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272507 T2446 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44877bf4 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272508 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272510 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272512 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node0
   [junit4]   2> 272515 T2440 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 272515 T2446 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 272516 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272517 T2448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f3ad51 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272518 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272519 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272521 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 272524 T2446 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 272524 T2448 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 272524 T2440 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 272525 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272527 T2450 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@553a7fa6 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272527 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272528 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272531 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node2
   [junit4]   2> 272534 T2450 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 272534 T2448 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 272534 T2440 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 272534 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272534 T2446 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 272536 T2452 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9f6ca59 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272536 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272538 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272540 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node3
   [junit4]   2> 272543 T2440 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 272543 T2450 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 272543 T2448 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 272543 T2452 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 272544 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272543 T2446 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 272546 T2454 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49b132e7 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272546 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272548 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272550 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node4
   [junit4]   2> 272553 T2440 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 272553 T2450 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 272553 T2452 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 272553 T2448 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 272553 T2446 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 272554 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272553 T2454 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 272556 T2456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d4d54d3 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272556 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272557 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272560 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node5
   [junit4]   2> 272563 T2446 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 272563 T2448 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 272563 T2450 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 272563 T2456 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 272563 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272563 T2454 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 272563 T2440 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 272563 T2452 oascc.ZkStateReader$3.process Updating live nodes... (6)
   [junit4]   2> 272565 T2458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3eb1579d name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272565 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272567 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272569 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node6
   [junit4]   2> 272572 T2452 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272572 T2448 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272573 T2454 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272573 T2440 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272573 T2450 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272572 T2456 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272572 T2446 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272573 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272573 T2458 oascc.ZkStateReader$3.process Updating live nodes... (7)
   [junit4]   2> 272575 T2460 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a923401 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272575 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272577 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272579 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node7
   [junit4]   2> 272582 T2450 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272582 T2458 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272583 T2440 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272583 T2452 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272583 T2456 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272582 T2448 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272582 T2454 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272582 T2446 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272582 T2460 oascc.ZkStateReader$3.process Updating live nodes... (8)
   [junit4]   2> 272583 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272585 T2462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26a183ac name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272585 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272587 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272589 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node8
   [junit4]   2> 272592 T2454 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272592 T2446 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272592 T2450 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272593 T2458 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272592 T2452 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272592 T2456 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272592 T2460 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272592 T2448 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272593 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272593 T2440 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272593 T2462 oascc.ZkStateReader$3.process Updating live nodes... (9)
   [junit4]   2> 272595 T2464 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@dfc9d74 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272595 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272597 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272599 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node9
   [junit4]   2> 272602 T2440 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272602 T2458 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2462 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2446 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2456 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2460 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272604 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272603 T2448 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2464 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2452 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2454 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272603 T2450 oascc.ZkStateReader$3.process Updating live nodes... (10)
   [junit4]   2> 272605 T2466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a85a2ab name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272606 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272608 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272610 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node10
   [junit4]   2> 272614 T2458 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2446 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2448 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2466 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2462 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2456 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272615 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272614 T2464 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2440 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272615 T2454 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2450 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2452 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272614 T2460 oascc.ZkStateReader$3.process Updating live nodes... (11)
   [junit4]   2> 272617 T2468 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e214713 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272617 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272619 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272621 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node11
   [junit4]   2> 272625 T2446 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2464 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2456 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2458 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2452 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2462 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2454 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2466 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2468 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272626 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272626 T2440 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2460 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2448 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272625 T2450 oascc.ZkStateReader$3.process Updating live nodes... (12)
   [junit4]   2> 272628 T2470 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@440303d8 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272628 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272630 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272632 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node12
   [junit4]   2> 272635 T2470 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2466 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2446 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2458 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272637 T2468 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2460 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2454 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2462 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272637 T2456 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272637 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272637 T2440 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2452 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2464 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2450 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272636 T2448 oascc.ZkStateReader$3.process Updating live nodes... (13)
   [junit4]   2> 272639 T2472 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@255eeb59 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272639 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272641 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272643 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node13
   [junit4]   2> 272646 T2464 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2452 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2458 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2446 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2462 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2468 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2440 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272648 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272648 T2466 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272648 T2460 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272648 T2472 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2470 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2454 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2448 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2450 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272647 T2456 oascc.ZkStateReader$3.process Updating live nodes... (14)
   [junit4]   2> 272650 T2474 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56c154ec name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272651 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272652 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272655 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node14
   [junit4]   2> 272658 T2450 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272658 T2472 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2460 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2452 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2474 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2446 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2440 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2468 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2456 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2470 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2466 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2462 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272658 T2448 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272658 T2454 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272660 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272659 T2464 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272659 T2458 oascc.ZkStateReader$3.process Updating live nodes... (15)
   [junit4]   2> 272662 T2476 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@218f0aff name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272663 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272664 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272667 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node15
   [junit4]   2> 272670 T2462 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272670 T2460 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272670 T2470 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2456 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2454 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2440 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2450 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2466 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2452 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2464 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2458 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2474 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2476 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2468 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2472 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2448 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272671 T2446 oascc.ZkStateReader$3.process Updating live nodes... (16)
   [junit4]   2> 272672 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272674 T2478 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ff74546 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272674 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272676 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272680 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node16
   [junit4]   2> 272687 T2458 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272687 T2452 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2450 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2448 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2474 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2472 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272687 T2466 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272689 T2476 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272689 T2460 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272689 T2468 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272689 T2454 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272689 T2464 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2478 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2456 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2440 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2470 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2446 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272688 T2462 oascc.ZkStateReader$3.process Updating live nodes... (17)
   [junit4]   2> 272691 T2480 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@69f5f01a name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272689 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272692 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272694 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272696 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node17
   [junit4]   2> 272699 T2468 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2446 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2456 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2472 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2454 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2440 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2470 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2462 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2458 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272702 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272701 T2480 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2464 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2450 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2478 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2452 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2460 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2476 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272701 T2474 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2466 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272700 T2448 oascc.ZkStateReader$3.process Updating live nodes... (18)
   [junit4]   2> 272704 T2482 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6bab5b13 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272705 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272706 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272708 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node18
   [junit4]   2> 272711 T2458 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272711 T2472 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2454 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2452 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2448 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2468 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272714 T2478 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2464 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272712 T2446 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272712 T2476 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272714 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272714 T2456 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2450 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2482 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2466 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2460 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2440 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2470 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2462 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2480 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272713 T2474 oascc.ZkStateReader$3.process Updating live nodes... (19)
   [junit4]   2> 272716 T2484 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7511d029 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272717 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272719 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272721 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node19
   [junit4]   2> 272724 T2472 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272725 T2466 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2450 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2470 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2454 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2440 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2464 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272727 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272726 T2482 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2484 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2452 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2478 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2480 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272725 T2474 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272725 T2456 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272729 T2486 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@752455d0 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272726 T2468 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2448 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2460 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2446 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2458 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2462 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272726 T2476 oascc.ZkStateReader$3.process Updating live nodes... (20)
   [junit4]   2> 272729 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272732 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272734 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node20
   [junit4]   2> 272738 T2462 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2446 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2484 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2486 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272740 T2478 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272738 T2458 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272740 T2456 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272740 T2468 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272740 T2452 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272740 T2466 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272740 T2440 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2464 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2480 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2448 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2482 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2474 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2460 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2470 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2450 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2476 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2472 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272739 T2454 oascc.ZkStateReader$3.process Updating live nodes... (21)
   [junit4]   2> 272742 T2488 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@720a3cc7 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272740 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272744 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272745 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272748 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node21
   [junit4]   2> 272751 T2440 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272752 T2446 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272752 T2456 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2484 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2488 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2448 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2472 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2476 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2482 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2466 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2486 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272754 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272754 T2478 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2454 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272754 T2460 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2464 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272756 T2490 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cb218c8 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272753 T2462 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2480 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2450 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2458 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2468 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2452 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2474 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272753 T2470 oascc.ZkStateReader$3.process Updating live nodes... (22)
   [junit4]   2> 272756 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272759 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272762 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node22
   [junit4]   2> 272766 T2472 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272766 T2476 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272766 T2480 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2464 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2448 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2458 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2440 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2466 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272766 T2468 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272766 T2488 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272766 T2484 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2460 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2490 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2482 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2454 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2474 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2456 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272768 T2446 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2450 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2478 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2452 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2470 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2462 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272767 T2486 oascc.ZkStateReader$3.process Updating live nodes... (23)
   [junit4]   2> 272770 T2492 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@359592e5 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272772 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272774 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272776 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node23
   [junit4]   2> 272780 T2470 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272780 T2462 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2486 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2460 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2448 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2450 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272783 T2472 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2454 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2488 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2452 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272781 T2476 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272781 T2474 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272781 T2456 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272781 T2466 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272780 T2490 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272783 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272783 T2482 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272783 T2492 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272783 T2484 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2446 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2478 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2468 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2458 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2464 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2440 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272782 T2480 oascc.ZkStateReader$3.process Updating live nodes... (24)
   [junit4]   2> 272785 T2494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@146bb749 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272787 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272788 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272791 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node24
   [junit4]   2> 272795 T2466 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272795 T2446 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2468 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2476 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2494 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2486 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2462 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272796 T2478 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272796 T2484 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272796 T2470 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272796 T2452 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272796 T2460 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272796 T2454 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272795 T2448 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272798 T2490 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2474 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2450 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2472 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2480 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2492 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272798 T2482 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2440 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2456 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2458 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2464 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272797 T2488 oascc.ZkStateReader$3.process Updating live nodes... (25)
   [junit4]   2> 272800 T2496 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36594c9d name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272803 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272804 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272806 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node25
   [junit4]   2> 272810 T2482 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272810 T2486 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272810 T2462 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2458 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272810 T2480 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2492 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2490 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272810 T2472 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2468 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2452 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2450 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2446 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2496 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2466 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2460 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2494 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2448 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2440 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272812 T2470 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2456 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2476 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2478 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2464 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2488 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2484 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2474 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272811 T2454 oascc.ZkStateReader$3.process Updating live nodes... (26)
   [junit4]   2> 272815 T2498 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d87cb0 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272813 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272817 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272819 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272821 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node26
   [junit4]   2> 272825 T2462 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272825 T2466 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2440 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2494 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2454 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2496 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2498 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2482 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2478 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272825 T2488 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272828 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272827 T2476 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2456 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2492 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2458 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2484 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2448 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272830 T2500 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a67bba4 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272827 T2460 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2486 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2480 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2474 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2452 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2472 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2464 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272827 T2446 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2468 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2450 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2470 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272826 T2490 oascc.ZkStateReader$3.process Updating live nodes... (27)
   [junit4]   2> 272830 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272834 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272836 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node27
   [junit4]   2> 272839 T2466 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272840 T2470 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272841 T2498 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272841 T2452 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2474 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2462 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272841 T2484 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272841 T2464 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272841 T2446 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272840 T2458 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272840 T2494 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272840 T2460 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272840 T2448 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272840 T2468 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272843 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272842 T2490 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272845 T2502 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d165570 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272842 T2500 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2492 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2480 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2478 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2472 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2454 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2450 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2476 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2486 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2488 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2496 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272842 T2482 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272841 T2456 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272841 T2440 oascc.ZkStateReader$3.process Updating live nodes... (28)
   [junit4]   2> 272845 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272849 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272851 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node28
   [junit4]   2> 272855 T2462 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272856 T2454 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272856 T2446 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272856 T2490 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2494 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2472 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2484 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2452 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2468 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2480 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2482 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2460 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2488 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2474 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272857 T2458 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272856 T2478 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272856 T2486 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272859 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272858 T2466 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2464 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2470 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272861 T2504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d2d62f1 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272858 T2448 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2492 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2502 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2450 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2476 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2456 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2498 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2496 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2500 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272858 T2440 oascc.ZkStateReader$3.process Updating live nodes... (29)
   [junit4]   2> 272862 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272865 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272867 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node29
   [junit4]   2> 272872 T2490 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2456 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2484 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2488 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272874 T2498 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272874 T2460 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272874 T2496 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272874 T2466 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272874 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272873 T2448 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2446 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2478 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2494 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2492 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2440 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2486 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2468 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272877 T2506 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c0d0687 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272873 T2470 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2472 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2462 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272873 T2450 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2458 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2480 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2452 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2454 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2482 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2474 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2502 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2476 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272872 T2500 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272877 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272874 T2504 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272874 T2464 oascc.ZkStateReader$3.process Updating live nodes... (30)
   [junit4]   2> 272881 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272883 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node30
   [junit4]   2> 272887 T2468 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272887 T2496 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2456 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2476 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2446 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2460 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2464 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2440 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2470 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2506 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272888 T2462 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2502 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2452 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2490 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2458 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2474 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2488 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2478 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272890 T2500 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2472 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2480 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2454 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2504 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2448 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2482 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2494 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2484 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2492 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2450 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2498 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2466 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272889 T2486 oascc.ZkStateReader$3.process Updating live nodes... (31)
   [junit4]   2> 272893 T2508 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@79c8eac7 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272891 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272895 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272897 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272899 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node31
   [junit4]   2> 272903 T2450 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272904 T2478 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272904 T2454 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2490 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272904 T2472 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2452 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2466 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272903 T2460 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272903 T2474 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2464 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2480 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2506 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2446 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2508 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2482 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2458 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2470 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272906 T2476 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272909 T2510 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d7ff7c0 name:ZooKeeperConnection Watcher:127.0.0.1:26676/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 272905 T2468 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2492 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2502 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2500 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2456 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2504 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2484 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2462 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2488 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2440 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2494 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2486 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272905 T2448 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272904 T2498 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272904 T2496 oascc.ZkStateReader$3.process Updating live nodes... (32)
   [junit4]   2> 272907 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 272912 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 272913 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 272916 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node32
   [junit4]   2> 272919 T2470 oascc.ZkStateReade

[...truncated too long message...]

t zk server on port:26874
   [junit4]   2> 316207 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316210 T2940 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60434618 name:ZooKeeperConnection Watcher:127.0.0.1:26874/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316211 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316212 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316214 T2942 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@106404a8 name:ZooKeeperConnection Watcher:127.0.0.1:26874 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316214 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316307 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316316 T2944 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dad1ac6 name:ZooKeeperConnection Watcher:127.0.0.1:26874 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316316 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316316 T2414 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 316320 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 316330 T2414 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 316332 T2414 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 316334 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 316336 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316342 T2946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@377b702a name:ZooKeeperConnection Watcher:127.0.0.1:26874/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316342 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316343 T2414 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 316345 T2414 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 316347 T2940 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 316347 T2946 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 316348 T2414 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 316349 T2948 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ec6953b name:ZooKeeperConnection Watcher:127.0.0.1:26874/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 316350 T2414 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 316350 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 316352 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 316356 T2414 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:26874_solr
   [junit4]   2> 316356 T2414 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 316358 T2414 oasc.Overseer.start Overseer (id=91431434935795716-127.0.0.1:26874_solr-n_0000000000) starting
   [junit4]   2> 316360 T2414 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 316363 T2414 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 316366 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 316369 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 316371 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 316376 T2414 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 316390 T2950 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 316399 T2950 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 316399 T2949 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 316401 T2949 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 316401 T2949 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"12",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 316402 T2949 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
   [junit4]   2> 316402 T2949 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard10
   [junit4]   2> 316404 T2948 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 316406 T2946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 316406 T2940 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 316901 T2414 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard10/election
   [junit4]   2> 316929 T2414 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard10
   [junit4]   2> 316935 T2948 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 316936 T2414 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 316936 T2949 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 316947 T2946 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 316947 T2940 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 316958 T2414 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:26874 26874
   [junit4]   2> 317075 T2414 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 317077 T2414 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 317077 T2414 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 320079 T2414 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene46: {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=fr_CA, timezone=Asia/Amman
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=144504728,total=335544320
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrXmlPersistence, DistributedDebugComponentTest, TestClassNameShortening, EchoParamsTest, TestDynamicFieldResource, TestCloudManagedSchemaAddField, TestReloadAndDeleteDocs, TestSerializedLuceneMatchVersion, TestZkChroot, TestDefaultSimilarityFactory, TestManagedSchemaFieldResource, TestExtendedDismaxParser, TestFaceting, NumericFieldsTest, DistributedQueryComponentOptimizationTest, SliceStateTest, TestCollationField, SolrIndexSplitterTest, TestRandomFaceting, TestSort, CoreAdminRequestStatusTest, TestUtils, ConnectionManagerTest, TestSearcherReuse, TestRandomMergePolicy, InfoHandlerTest, SpellCheckComponentTest, TestSolrDeletionPolicy2, TermVectorComponentDistributedTest, OverseerTest]
   [junit4] Completed on J0 in 48.37s, 9 tests, 1 error <<< FAILURES!

[...truncated 987 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:467: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:447: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:45: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:490: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1276: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:908: There were test failures: 376 suites, 1595 tests, 1 error, 37 ignored (16 assumptions)

Total time: 74 minutes 28 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure