You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/01/03 18:08:20 UTC

[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.7.0_60-ea-b02) - Build # 8943 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/8943/
Java: 64bit/jdk1.7.0_60-ea-b02 -XX:+UseCompressedOops -XX:+UseParallelGC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerTest.testOverseerFailure

Error Message:
KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1

Stack Trace:
org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
	at __randomizedtesting.SeedInfo.seed([F59CCED46009798A:F194412772AC96AB]:0)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:428)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:425)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)


REGRESSION:  org.apache.solr.cloud.OverseerTest.testStateChange

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([F59CCED46009798A:D618DAF5BC2A985D]:0)
	at org.apache.solr.cloud.OverseerTest.testStateChange(OverseerTest.java:469)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)




Build Log:
[...truncated 10440 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerTest
   [junit4]   2> 344369 T1275 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-OverseerTest-1388768121854
   [junit4]   2> 344369 T1275 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 344372 T1275 oas.SolrTestCaseJ4.setUp ###Starting testOverseerFailure
   [junit4]   2> 344372 T1275 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 344373 T1276 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 344473 T1275 oasc.ZkTestServer.run start zk server on port:53407
   [junit4]   2> 344474 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 344594 T1283 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e71eb20 name:ZooKeeperConnection Watcher:127.0.0.1:53407/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 344595 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 344596 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 344598 T1286 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@141c08d2 name:ZooKeeperConnection Watcher:127.0.0.1:53407 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 344598 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 344600 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 344601 T1289 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30f36d6e name:ZooKeeperConnection Watcher:127.0.0.1:53407 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 344602 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 344602 T1275 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 344604 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 344605 T1277 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1435907d4040002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 344609 T1275 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 344612 T1275 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 344615 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 344618 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 344619 T1292 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@356ac7a3 name:ZooKeeperConnection Watcher:127.0.0.1:53407/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 344621 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 344622 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 344624 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 344626 T1283 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 344626 T1292 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 344626 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 344628 T1295 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2da4ea33 name:ZooKeeperConnection Watcher:127.0.0.1:53407/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 344628 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 344629 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 344630 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 344633 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 344634 T1275 oasc.Overseer.start Overseer (id=91014307634216964-127.0.0.1:53407_solr-n_0000000000) starting
   [junit4]   2> 344636 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 344638 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 344640 T1275 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 344641 T1297 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 344642 T1296 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 345644 T1295 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 345646 T1296 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 345646 T1296 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 345646 T1296 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 345646 T1296 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 345648 T1295 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 345649 T1292 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 345649 T1283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 346144 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 346156 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 346161 T1295 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 346162 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 346163 T1296 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 346164 T1295 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 346164 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 346167 T1296 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 346168 T1296 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 346172 T1295 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 346274 T1283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 346274 T1292 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 346326 T1277 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1435907d4040004, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 347328 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 347334 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347335 T1300 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2517e9f4 name:ZooKeeperConnection Watcher:127.0.0.1:53407/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347335 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347338 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 347340 T1275 oasc.Overseer.start Overseer (id=91014307634216965-127.0.0.1:53407_solr-n_0000000001) starting
   [junit4]   2> 347346 T1302 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 347347 T1301 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 347350 T1301 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 347353 T1301 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 347353 T1301 oasc.Overseer$ClusterStateUpdater.updateState shard=shard1 is already registered
   [junit4]   2> 347358 T1300 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 347359 T1283 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 347359 T1292 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 347415 T1300 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 347415 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 347417 T1301 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 347426 T1275 oasc.LeaderElector.checkIfIamLeader WARN Failed setting watch org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election/91014307634216963-node1_core1-n_0000000001
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1151)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:273)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:270)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:270)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:126)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 347428 T1292 oasc.LeaderElector.checkIfIamLeader WARN Our node is no longer in line to be leader
   [junit4]   2> 347428 T1275 oasc.LeaderElector.checkIfIamLeader WARN Failed setting watch org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /collections/collection1/leader_elect/shard1/election/91014307634216963-node1_core1-n_0000000000
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:111)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1151)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:273)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:270)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:270)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:126)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 347430 T1301 oasc.SolrException.log ERROR Problem cleaning up collection in zk:collection1:org.apache.zookeeper.KeeperException$NotEmptyException: KeeperErrorCode = Directory not empty for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:125)
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 		at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:173)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:170)
   [junit4]   2> 		at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:170)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:654)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.clean(SolrZkClient.java:649)
   [junit4]   2> 		at org.apache.solr.cloud.Overseer$ClusterStateUpdater.removeCore(Overseer.java:875)
   [junit4]   2> 		at org.apache.solr.cloud.Overseer$ClusterStateUpdater.processMessage(Overseer.java:226)
   [junit4]   2> 		at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:187)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 	
   [junit4]   2> 347430 T1292 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 347431 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 347432 T1300 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 347434 T1283 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 347437 T1275 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:53407 53407
   [junit4]   2> 347533 T1301 oasc.Overseer$ClusterStateUpdater.run WARN Solr cannot talk to ZK, exiting Overseer main queue loop org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.orderedChildren(DistributedQueue.java:89)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:411)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:195)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 347624 T1275 oas.SolrTestCaseJ4.tearDown ###Ending testOverseerFailure
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=OverseerTest -Dtests.method=testOverseerFailure -Dtests.seed=F59CCED46009798A -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr_RS_#Latn -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.file.encoding=UTF-8
   [junit4] ERROR   3.26s J0 | OverseerTest.testOverseerFailure <<<
   [junit4]    > Throwable #1: org.apache.zookeeper.KeeperException$NodeExistsException: KeeperErrorCode = NodeExists for /collections/collection1/leaders/shard1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F59CCED46009798A:F194412772AC96AB]:0)
   [junit4]    > 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:119)
   [junit4]    > 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]    > 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient$10.execute(SolrZkClient.java:428)
   [junit4]    > 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:425)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
   [junit4]    > 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
   [junit4]    > 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:157)
   [junit4]    > 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:277)
   [junit4]    > 	at org.apache.solr.cloud.OverseerTest$MockZKController.publishState(OverseerTest.java:153)
   [junit4]    > 	at org.apache.solr.cloud.OverseerTest.testOverseerFailure(OverseerTest.java:584)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 347638 T1275 oas.SolrTestCaseJ4.setUp ###Starting testPlaceholders
   [junit4]   2> 347638 T1275 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 347639 T1303 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 347740 T1275 oasc.ZkTestServer.run start zk server on port:37710
   [junit4]   2> 347741 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347744 T1310 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@671e5126 name:ZooKeeperConnection Watcher:127.0.0.1:37710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347745 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347746 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347747 T1313 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20fe8dc5 name:ZooKeeperConnection Watcher:127.0.0.1:37710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347747 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347765 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347766 T1316 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4466c671 name:ZooKeeperConnection Watcher:127.0.0.1:37710 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347767 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347767 T1275 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 347769 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 347771 T1275 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 347772 T1275 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 347775 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 347777 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347779 T1319 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10bb55b8 name:ZooKeeperConnection Watcher:127.0.0.1:37710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347779 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347780 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 347782 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 347784 T1310 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 347784 T1319 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 347784 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 347785 T1322 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24664e24 name:ZooKeeperConnection Watcher:127.0.0.1:37710/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 347786 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 347786 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 347788 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 347790 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 347792 T1275 oasc.Overseer.start Overseer (id=91014307848716292-127.0.0.1:37710_solr-n_0000000000) starting
   [junit4]   2> 347793 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 347795 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 347797 T1275 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 347799 T1324 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 347800 T1323 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 347802 T1323 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 347802 T1323 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=12 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "node_name":"node1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"12",
   [junit4]   2> 	  "base_url":"http://node1/solr/"}
   [junit4]   2> 347802 T1323 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8, shard9, shard10, shard11, shard12]
   [junit4]   2> 347803 T1323 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard4
   [junit4]   2> 347804 T1322 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 347805 T1319 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 347805 T1310 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 348301 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard4/election
   [junit4]   2> 348310 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard4
   [junit4]   2> 348315 T1322 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 348316 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 348316 T1323 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 348329 T1310 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 348329 T1319 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 348331 T1275 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:37710 37710
   [junit4]   2> 348546 T1275 oas.SolrTestCaseJ4.tearDown ###Ending testPlaceholders
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 348552 T1275 oas.SolrTestCaseJ4.setUp ###Starting testShardAssignment
   [junit4]   2> 348552 T1275 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 348552 T1325 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 348652 T1275 oasc.ZkTestServer.run start zk server on port:40815
   [junit4]   2> 348653 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 348656 T1332 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fc3a407 name:ZooKeeperConnection Watcher:127.0.0.1:40815 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 348656 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 348677 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 348678 T1335 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71cd8586 name:ZooKeeperConnection Watcher:127.0.0.1:40815 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 348678 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 348679 T1275 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 348682 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 348683 T1338 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64779bab name:ZooKeeperConnection Watcher:127.0.0.1:40815/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 348683 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 348683 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 348685 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 348686 T1341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8f641e4 name:ZooKeeperConnection Watcher:127.0.0.1:40815/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 348686 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 348687 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 348689 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 348692 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 348693 T1275 oasc.Overseer.start Overseer (id=91014307908288515-127.0.0.1:40815_solr-n_0000000000) starting
   [junit4]   2> 348695 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 348697 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 348699 T1275 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 348701 T1343 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 348701 T1275 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 348702 T1342 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 348703 T1275 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 348705 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 348708 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 348709 T1346 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b96ed83 name:ZooKeeperConnection Watcher:127.0.0.1:40815/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 348710 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 348711 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 348715 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1
   [junit4]   2> 348719 T1338 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 348720 T1346 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 348721 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 348725 T1342 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 348726 T1342 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1",
   [junit4]   2> 	  "core":"core1",
   [junit4]   2> 	  "core_node_name":"node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr/"}
   [junit4]   2> 348726 T1342 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 348726 T1342 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 348729 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 348729 T1338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 348729 T1346 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 348935 T1292 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue/qn-
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.create(ZooKeeper.java:783)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:239)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$5.execute(SolrZkClient.java:236)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.create(SolrZkClient.java:236)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.createData(DistributedQueue.java:284)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.offer(DistributedQueue.java:271)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:121)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 348935 T1292 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 349223 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 349232 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 349238 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349241 T1342 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 349242 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349245 T1342 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1",
   [junit4]   2> 	  "core":"core2",
   [junit4]   2> 	  "core_node_name":"node2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr/"}
   [junit4]   2> 349245 T1342 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 349245 T1342 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 349251 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349353 T1338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 349353 T1346 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 349742 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 349748 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 349752 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349752 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349752 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349756 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349756 T1342 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 349760 T1342 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1",
   [junit4]   2> 	  "core":"core3",
   [junit4]   2> 	  "core_node_name":"node3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr/"}
   [junit4]   2> 349760 T1342 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 349760 T1342 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 349762 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 349864 T1338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 349865 T1346 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350256 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 350263 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 350268 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350269 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350269 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350270 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350270 T1342 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 350273 T1342 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1",
   [junit4]   2> 	  "core":"core4",
   [junit4]   2> 	  "core_node_name":"node4",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr/"}
   [junit4]   2> 350273 T1342 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 350274 T1342 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 350276 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350379 T1346 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350379 T1338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350770 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 350781 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350781 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350781 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350783 T1342 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 350783 T1342 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1",
   [junit4]   2> 	  "core":"core5",
   [junit4]   2> 	  "core_node_name":"node5",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr/"}
   [junit4]   2> 350784 T1342 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 350784 T1342 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 350786 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 350888 T1338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 350888 T1346 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 351278 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 351284 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 351284 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 351284 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 351285 T1342 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 351286 T1342 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1",
   [junit4]   2> 	  "core":"core6",
   [junit4]   2> 	  "core_node_name":"node6",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1/solr/"}
   [junit4]   2> 351286 T1342 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 351286 T1342 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 351289 T1341 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 351391 T1338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 351391 T1346 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 351784 T1275 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 351792 T1346 oascc.ZkStateReader$3.process Updating live nodes... (0)
   [junit4]   2> 351794 T1275 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:40815 40815
   [junit4]   2> 352045 T1275 oas.SolrTestCaseJ4.tearDown ###Ending testShardAssignment
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 352050 T1275 oas.SolrTestCaseJ4.setUp ###Starting testStateChange
   [junit4]   2> 352050 T1275 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 352051 T1347 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 352151 T1275 oasc.ZkTestServer.run start zk server on port:57071
   [junit4]   2> 352152 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352155 T1354 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6af44b39 name:ZooKeeperConnection Watcher:127.0.0.1:57071/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352156 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352156 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352164 T1357 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1de81bc9 name:ZooKeeperConnection Watcher:127.0.0.1:57071 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352164 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352183 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352185 T1360 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@33932f51 name:ZooKeeperConnection Watcher:127.0.0.1:57071 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352185 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352185 T1275 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 352187 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 352189 T1275 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 352190 T1275 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 352191 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 352193 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352194 T1363 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f2b413b name:ZooKeeperConnection Watcher:127.0.0.1:57071/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352194 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352195 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 352196 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 352199 T1275 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 352200 T1275 oasc.Overseer.start Overseer (id=91014308137795587-127.0.0.1:57071_solr-n_0000000000) starting
   [junit4]   2> 352202 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 352204 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 352206 T1275 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 352208 T1365 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 352210 T1364 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 352212 T1363 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 352212 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352263 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352315 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352366 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352418 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352469 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352520 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352572 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352624 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352676 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352728 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352779 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352831 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352883 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352935 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 352986 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353037 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353089 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353141 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353192 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353245 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353292 T1346 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 353293 T1346 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 353294 T1346 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 353296 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353348 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353399 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353451 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353502 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353554 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353606 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353657 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353709 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353760 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353812 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353863 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353915 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 353967 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354018 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354069 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354122 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354174 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354225 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354277 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354328 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354380 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354431 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354483 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354534 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354586 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354637 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354688 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354740 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354791 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354842 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354894 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354946 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 354997 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355048 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355100 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355152 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355204 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355256 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355307 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355359 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355411 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355462 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355513 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355565 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355617 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355668 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355720 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355771 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355823 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355874 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355927 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 355978 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356030 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356081 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356133 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356185 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356237 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356288 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356340 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356391 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356443 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356494 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356546 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356598 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356649 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356700 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356752 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356803 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356855 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356906 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356958 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357009 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357061 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357112 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357164 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357215 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357267 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357319 T1275 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357369 T1275 oasc.OverseerTest.waitForCollections WARN Timeout waiting for collections: [collection1] state:live nodes:[] collections:{}
   [junit4]   2> 357373 T1275 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:57071 57071
   [junit4]   2> 357545 T1275 oas.SolrTestCaseJ4.tearDown ###Ending testStateChange
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=OverseerTest -Dtests.method=testStateChange -Dtests.seed=F59CCED46009798A -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr_RS_#Latn -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.file.encoding=UTF-8
   [junit4] ERROR   5.51s J0 | OverseerTest.testStateChange <<<
   [junit4]    > Throwable #1: java.lang.NullPointerException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F59CCED46009798A:D618DAF5BC2A985D]:0)
   [junit4]    > 	at org.apache.solr.cloud.OverseerTest.testStateChange(OverseerTest.java:469)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 357559 T1275 oas.SolrTestCaseJ4.setUp ###Starting testShardLeaderChange
   [junit4]   2> 357559 T1275 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 357560 T1366 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 357660 T1275 oasc.ZkTestServer.run start zk server on port:59265
   [junit4]   2> 357661 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357664 T1373 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e2f9c63 name:ZooKeeperConnection Watcher:127.0.0.1:59265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357664 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357665 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357666 T1376 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@417f9dc name:ZooKeeperConnection Watcher:127.0.0.1:59265 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357667 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357682 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357684 T1379 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@32e8c7f8 name:ZooKeeperConnection Watcher:127.0.0.1:59265 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357685 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357685 T1275 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 357688 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 357691 T1380 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357691 T1275 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357692 T1385 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d759c25 name:ZooKeeperConnection Watcher:127.0.0.1:59265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357693 T1380 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357693 T1386 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@225df360 name:ZooKeeperConnection Watcher:127.0.0.1:59265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357693 T1275 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357693 T1380 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 357694 T1275 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 357696 T1380 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 357696 T1275 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 357699 T1275 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 357700 T1380 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 357703 T1275 oascc.SolrZkClient.makePath makePath: /live_nodes/node1
   [junit4]   2> 357703 T1380 oasc.Overseer.start Overseer (id=91014308498767875-127.0.0.1:59265_solr-n_0000000000) starting
   [junit4]   2> 357705 T1386 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 357706 T1380 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 357706 T1275 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 357709 T1380 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 357715 T1380 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-w

[...truncated too long message...]

ientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 371370 T1747 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 371373 T1750 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 371391 T1759 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1041)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:205)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:202)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:202)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:413)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 371391 T1759 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 371408 T1771 oasc.LeaderElector$1.process WARN  org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.<init>(DistributedQueue.java:63)
   [junit4]   2> 	at org.apache.solr.cloud.Overseer.getInQueue(Overseer.java:1020)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:121)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer/queue
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1041)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:222)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$4.execute(SolrZkClient.java:219)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:219)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:100)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.ensureExists(ZkCmdExecutor.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.DistributedQueue.<init>(DistributedQueue.java:61)
   [junit4]   2> 	... 8 more
   [junit4]   2> 
   [junit4]   2> 371409 T1771 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 371414 T1777 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1041)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:205)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:202)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:202)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:413)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:382)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:369)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:112)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:165)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:109)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 371414 T1777 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 373045 T1275 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=sr_RS_#Latn, timezone=America/Argentina/Rio_Gallegos
   [junit4]   2> NOTE: Linux 3.8.0-34-generic amd64/Oracle Corporation 1.7.0_60-ea (64-bit)/cpus=8,threads=1,free=73974000,total=276824064
   [junit4]   2> NOTE: All tests run in this JVM: [BadComponentTest, TestPostingsSolrHighlighter, CircularListTest, TestPHPSerializedResponseWriter, ResourceLoaderTest, AutoCommitTest, TestReversedWildcardFilterFactory, DeleteReplicaTest, TestDynamicFieldResource, TestSolrXMLSerializer, TestDocBasedVersionConstraints, EnumFieldTest, PrimitiveFieldTypeTest, ShowFileRequestHandlerTest, SynonymTokenizerTest, ScriptEngineTest, SoftAutoCommitTest, DocValuesMissingTest, FastVectorHighlighterTest, TestSort, StatelessScriptUpdateProcessorFactoryTest, SliceStateUpdateTest, DistributedSpellCheckComponentTest, SuggestComponentTest, RecoveryZkTest, RegexBoostProcessorTest, TestHighFrequencyDictionaryFactory, TestRemoteStreaming, TestWriterPerf, MoreLikeThisHandlerTest, HdfsSyncSliceTest, SampleTest, TestCollationFieldDocValues, BJQParserTest, InfoHandlerTest, TestJoin, LukeRequestHandlerTest, TestBadConfig, DeleteShardTest, TestSurroundQueryParser, URLClassifyProcessorTest, TestSolrQueryParser, TestConfig, SystemInfoHandlerTest, TestPseudoReturnFields, TestInfoStreamLogging, CoreAdminCreateDiscoverTest, TestLRUCache, CoreMergeIndexesAdminHandlerTest, EchoParamsTest, TestExtendedDismaxParser, TestSearchPerf, SimpleFacetsTest, TestStressVersions, SpellPossibilityIteratorTest, TermsComponentTest, SliceStateTest, UnloadDistributedZkTest, ExternalFileFieldSortTest, HdfsBasicDistributedZkTest, IndexSchemaTest, ClusterStateUpdateTest, CoreAdminHandlerTest, TestDynamicFieldCollectionResource, SearchHandlerTest, NoFacetTest, TestSchemaSimilarityResource, HdfsDirectoryTest, HdfsBasicDistributedZk2Test, OverseerTest]
   [junit4] Completed on J0 in 28.72s, 8 tests, 2 errors <<< FAILURES!

[...truncated 647 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:453: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:433: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:491: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1304: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:937: There were test failures: 358 suites, 1577 tests, 2 errors, 35 ignored (6 assumptions)

Total time: 40 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0_60-ea-b02 -XX:+UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-trunk-Linux (64bit/ibm-j9-jdk7) - Build # 8945 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/8945/
Java: 64bit/ibm-j9-jdk7 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}

6 tests failed.
REGRESSION:  org.apache.solr.analytics.expression.ExpressionTest.addTest

Error Message:
Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='ar']/double[@name='sum']  raw data: <?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">0</int>   <int name="QTime">7</int> </lst> <result name="response" numFound="100" start="0"> </result> <lst name="stats">   <lst name="stats_fields"/> </lst> <lst name="stats">   <lst name="nr">     <double name="c">-100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="rr">     <str name="max">str9</str>     <str name="min">str0</str>     <str name="rmax">9rts</str>     <str name="rmin">0rts</str>   </lst>   <lst name="cdr">     <date name="cd1">1800-12-31T23:59:59Z</date>     <date name="cd2">1804-06-30T23:59:59Z</date>     <str name="cs1">1800-12-31T23:59:59Z</str>     <str name="cs2">1804-06-30T23:59:59Z</str>   </lst>   <lst name="dr">     <long name="count">100</long>     <double name="mc">0.2891</double>     <double name="mean">28.91</double>     <double name="su">80.30555555555556</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="mr">     <long name="count">100</long>     <double name="mcm">70829.5</double>     <double name="mean">28.91</double>     <double name="median">24.5</double>     <double name="su">104076.0</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="avr">     <double name="c">100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="ar">     <long name="count">100</long>     <null name="mcm"/>     <null name="mean"/>     <null name="median"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst>   <lst name="csr">     <str name="cs1">this is the first</str>     <str name="cs2">this is the second</str>     <str name="cs3">this is the third</str>   </lst>   <lst name="cr">     <str name="ccmax">this is the secondstr9</str>     <str name="ccmin">this is the firststr0</str>     <str name="csmax">this is the second</str>     <str name="csmin">this is the first</str>     <str name="max">str9</str>     <str name="min">str0</str>   </lst>   <lst name="dmr">     <str name="cma">+2MONTHS</str>     <str name="cme">+2YEARS</str>     <date name="dmma">1812-02-29T23:59:59Z</date>     <date name="dmme">1807-12-31T23:59:59Z</date>     <date name="max">1811-12-31T23:59:59Z</date>     <date name="median">1805-12-31T23:59:59Z</date>   </lst>   <lst name="cnr">     <double name="c10">10.0</double>     <double name="c8">8.0</double>   </lst>   <lst name="pr">     <long name="count">100</long>     <null name="mc"/>     <null name="mean"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst> </lst> </response> 

Stack Trace:
java.lang.AssertionError: Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='ar']/double[@name='sum'] 
raw data: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">7</int>
</lst>
<result name="response" numFound="100" start="0">
</result>
<lst name="stats">
  <lst name="stats_fields"/>
</lst>
<lst name="stats">
  <lst name="nr">
    <double name="c">-100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="rr">
    <str name="max">str9</str>
    <str name="min">str0</str>
    <str name="rmax">9rts</str>
    <str name="rmin">0rts</str>
  </lst>
  <lst name="cdr">
    <date name="cd1">1800-12-31T23:59:59Z</date>
    <date name="cd2">1804-06-30T23:59:59Z</date>
    <str name="cs1">1800-12-31T23:59:59Z</str>
    <str name="cs2">1804-06-30T23:59:59Z</str>
  </lst>
  <lst name="dr">
    <long name="count">100</long>
    <double name="mc">0.2891</double>
    <double name="mean">28.91</double>
    <double name="su">80.30555555555556</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="mr">
    <long name="count">100</long>
    <double name="mcm">70829.5</double>
    <double name="mean">28.91</double>
    <double name="median">24.5</double>
    <double name="su">104076.0</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="avr">
    <double name="c">100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="ar">
    <long name="count">100</long>
    <null name="mcm"/>
    <null name="mean"/>
    <null name="median"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
  <lst name="csr">
    <str name="cs1">this is the first</str>
    <str name="cs2">this is the second</str>
    <str name="cs3">this is the third</str>
  </lst>
  <lst name="cr">
    <str name="ccmax">this is the secondstr9</str>
    <str name="ccmin">this is the firststr0</str>
    <str name="csmax">this is the second</str>
    <str name="csmin">this is the first</str>
    <str name="max">str9</str>
    <str name="min">str0</str>
  </lst>
  <lst name="dmr">
    <str name="cma">+2MONTHS</str>
    <str name="cme">+2YEARS</str>
    <date name="dmma">1812-02-29T23:59:59Z</date>
    <date name="dmme">1807-12-31T23:59:59Z</date>
    <date name="max">1811-12-31T23:59:59Z</date>
    <date name="median">1805-12-31T23:59:59Z</date>
  </lst>
  <lst name="cnr">
    <double name="c10">10.0</double>
    <double name="c8">8.0</double>
  </lst>
  <lst name="pr">
    <long name="count">100</long>
    <null name="mc"/>
    <null name="mean"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
</lst>
</response>

	at __randomizedtesting.SeedInfo.seed([9989DBAC881D3AF4:6FA57672C9D23DB3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.analytics.AbstractAnalyticsStatsTest.getStatResult(AbstractAnalyticsStatsTest.java:113)
	at org.apache.solr.analytics.expression.ExpressionTest.addTest(ExpressionTest.java:78)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
	at java.lang.reflect.Method.invoke(Method.java:613)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:780)


REGRESSION:  org.apache.solr.analytics.expression.ExpressionTest.absoluteValueTest

Error Message:
Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='avr']/double[@name='sum']  raw data: <?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">0</int>   <int name="QTime">7</int> </lst> <result name="response" numFound="100" start="0"> </result> <lst name="stats">   <lst name="stats_fields"/> </lst> <lst name="stats">   <lst name="nr">     <double name="c">-100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="rr">     <str name="max">str9</str>     <str name="min">str0</str>     <str name="rmax">9rts</str>     <str name="rmin">0rts</str>   </lst>   <lst name="cdr">     <date name="cd1">1800-12-31T23:59:59Z</date>     <date name="cd2">1804-06-30T23:59:59Z</date>     <str name="cs1">1800-12-31T23:59:59Z</str>     <str name="cs2">1804-06-30T23:59:59Z</str>   </lst>   <lst name="dr">     <long name="count">100</long>     <double name="mc">0.2891</double>     <double name="mean">28.91</double>     <double name="su">80.30555555555556</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="mr">     <long name="count">100</long>     <double name="mcm">70829.5</double>     <double name="mean">28.91</double>     <double name="median">24.5</double>     <double name="su">104076.0</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="avr">     <double name="c">100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="ar">     <long name="count">100</long>     <null name="mcm"/>     <null name="mean"/>     <null name="median"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst>   <lst name="csr">     <str name="cs1">this is the first</str>     <str name="cs2">this is the second</str>     <str name="cs3">this is the third</str>   </lst>   <lst name="cr">     <str name="ccmax">this is the secondstr9</str>     <str name="ccmin">this is the firststr0</str>     <str name="csmax">this is the second</str>     <str name="csmin">this is the first</str>     <str name="max">str9</str>     <str name="min">str0</str>   </lst>   <lst name="dmr">     <str name="cma">+2MONTHS</str>     <str name="cme">+2YEARS</str>     <date name="dmma">1812-02-29T23:59:59Z</date>     <date name="dmme">1807-12-31T23:59:59Z</date>     <date name="max">1811-12-31T23:59:59Z</date>     <date name="median">1805-12-31T23:59:59Z</date>   </lst>   <lst name="cnr">     <double name="c10">10.0</double>     <double name="c8">8.0</double>   </lst>   <lst name="pr">     <long name="count">100</long>     <null name="mc"/>     <null name="mean"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst> </lst> </response> 

Stack Trace:
java.lang.AssertionError: Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='avr']/double[@name='sum'] 
raw data: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">7</int>
</lst>
<result name="response" numFound="100" start="0">
</result>
<lst name="stats">
  <lst name="stats_fields"/>
</lst>
<lst name="stats">
  <lst name="nr">
    <double name="c">-100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="rr">
    <str name="max">str9</str>
    <str name="min">str0</str>
    <str name="rmax">9rts</str>
    <str name="rmin">0rts</str>
  </lst>
  <lst name="cdr">
    <date name="cd1">1800-12-31T23:59:59Z</date>
    <date name="cd2">1804-06-30T23:59:59Z</date>
    <str name="cs1">1800-12-31T23:59:59Z</str>
    <str name="cs2">1804-06-30T23:59:59Z</str>
  </lst>
  <lst name="dr">
    <long name="count">100</long>
    <double name="mc">0.2891</double>
    <double name="mean">28.91</double>
    <double name="su">80.30555555555556</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="mr">
    <long name="count">100</long>
    <double name="mcm">70829.5</double>
    <double name="mean">28.91</double>
    <double name="median">24.5</double>
    <double name="su">104076.0</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="avr">
    <double name="c">100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="ar">
    <long name="count">100</long>
    <null name="mcm"/>
    <null name="mean"/>
    <null name="median"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
  <lst name="csr">
    <str name="cs1">this is the first</str>
    <str name="cs2">this is the second</str>
    <str name="cs3">this is the third</str>
  </lst>
  <lst name="cr">
    <str name="ccmax">this is the secondstr9</str>
    <str name="ccmin">this is the firststr0</str>
    <str name="csmax">this is the second</str>
    <str name="csmin">this is the first</str>
    <str name="max">str9</str>
    <str name="min">str0</str>
  </lst>
  <lst name="dmr">
    <str name="cma">+2MONTHS</str>
    <str name="cme">+2YEARS</str>
    <date name="dmma">1812-02-29T23:59:59Z</date>
    <date name="dmme">1807-12-31T23:59:59Z</date>
    <date name="max">1811-12-31T23:59:59Z</date>
    <date name="median">1805-12-31T23:59:59Z</date>
  </lst>
  <lst name="cnr">
    <double name="c10">10.0</double>
    <double name="c8">8.0</double>
  </lst>
  <lst name="pr">
    <long name="count">100</long>
    <null name="mc"/>
    <null name="mean"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
</lst>
</response>

	at __randomizedtesting.SeedInfo.seed([9989DBAC881D3AF4:B825FBB945806292]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.analytics.AbstractAnalyticsStatsTest.getStatResult(AbstractAnalyticsStatsTest.java:113)
	at org.apache.solr.analytics.expression.ExpressionTest.absoluteValueTest(ExpressionTest.java:143)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
	at java.lang.reflect.Method.invoke(Method.java:613)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:780)


REGRESSION:  org.apache.solr.analytics.expression.ExpressionTest.negateTest

Error Message:
Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='nr']/double[@name='sum']  raw data: <?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">0</int>   <int name="QTime">7</int> </lst> <result name="response" numFound="100" start="0"> </result> <lst name="stats">   <lst name="stats_fields"/> </lst> <lst name="stats">   <lst name="nr">     <double name="c">-100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="rr">     <str name="max">str9</str>     <str name="min">str0</str>     <str name="rmax">9rts</str>     <str name="rmin">0rts</str>   </lst>   <lst name="cdr">     <date name="cd1">1800-12-31T23:59:59Z</date>     <date name="cd2">1804-06-30T23:59:59Z</date>     <str name="cs1">1800-12-31T23:59:59Z</str>     <str name="cs2">1804-06-30T23:59:59Z</str>   </lst>   <lst name="dr">     <long name="count">100</long>     <double name="mc">0.2891</double>     <double name="mean">28.91</double>     <double name="su">80.30555555555556</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="mr">     <long name="count">100</long>     <double name="mcm">70829.5</double>     <double name="mean">28.91</double>     <double name="median">24.5</double>     <double name="su">104076.0</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="avr">     <double name="c">100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="ar">     <long name="count">100</long>     <null name="mcm"/>     <null name="mean"/>     <null name="median"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst>   <lst name="csr">     <str name="cs1">this is the first</str>     <str name="cs2">this is the second</str>     <str name="cs3">this is the third</str>   </lst>   <lst name="cr">     <str name="ccmax">this is the secondstr9</str>     <str name="ccmin">this is the firststr0</str>     <str name="csmax">this is the second</str>     <str name="csmin">this is the first</str>     <str name="max">str9</str>     <str name="min">str0</str>   </lst>   <lst name="dmr">     <str name="cma">+2MONTHS</str>     <str name="cme">+2YEARS</str>     <date name="dmma">1812-02-29T23:59:59Z</date>     <date name="dmme">1807-12-31T23:59:59Z</date>     <date name="max">1811-12-31T23:59:59Z</date>     <date name="median">1805-12-31T23:59:59Z</date>   </lst>   <lst name="cnr">     <double name="c10">10.0</double>     <double name="c8">8.0</double>   </lst>   <lst name="pr">     <long name="count">100</long>     <null name="mc"/>     <null name="mean"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst> </lst> </response> 

Stack Trace:
java.lang.AssertionError: Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='nr']/double[@name='sum'] 
raw data: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">7</int>
</lst>
<result name="response" numFound="100" start="0">
</result>
<lst name="stats">
  <lst name="stats_fields"/>
</lst>
<lst name="stats">
  <lst name="nr">
    <double name="c">-100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="rr">
    <str name="max">str9</str>
    <str name="min">str0</str>
    <str name="rmax">9rts</str>
    <str name="rmin">0rts</str>
  </lst>
  <lst name="cdr">
    <date name="cd1">1800-12-31T23:59:59Z</date>
    <date name="cd2">1804-06-30T23:59:59Z</date>
    <str name="cs1">1800-12-31T23:59:59Z</str>
    <str name="cs2">1804-06-30T23:59:59Z</str>
  </lst>
  <lst name="dr">
    <long name="count">100</long>
    <double name="mc">0.2891</double>
    <double name="mean">28.91</double>
    <double name="su">80.30555555555556</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="mr">
    <long name="count">100</long>
    <double name="mcm">70829.5</double>
    <double name="mean">28.91</double>
    <double name="median">24.5</double>
    <double name="su">104076.0</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="avr">
    <double name="c">100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="ar">
    <long name="count">100</long>
    <null name="mcm"/>
    <null name="mean"/>
    <null name="median"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
  <lst name="csr">
    <str name="cs1">this is the first</str>
    <str name="cs2">this is the second</str>
    <str name="cs3">this is the third</str>
  </lst>
  <lst name="cr">
    <str name="ccmax">this is the secondstr9</str>
    <str name="ccmin">this is the firststr0</str>
    <str name="csmax">this is the second</str>
    <str name="csmin">this is the first</str>
    <str name="max">str9</str>
    <str name="min">str0</str>
  </lst>
  <lst name="dmr">
    <str name="cma">+2MONTHS</str>
    <str name="cme">+2YEARS</str>
    <date name="dmma">1812-02-29T23:59:59Z</date>
    <date name="dmme">1807-12-31T23:59:59Z</date>
    <date name="max">1811-12-31T23:59:59Z</date>
    <date name="median">1805-12-31T23:59:59Z</date>
  </lst>
  <lst name="cnr">
    <double name="c10">10.0</double>
    <double name="c8">8.0</double>
  </lst>
  <lst name="pr">
    <long name="count">100</long>
    <null name="mc"/>
    <null name="mean"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
</lst>
</response>

	at __randomizedtesting.SeedInfo.seed([9989DBAC881D3AF4:9181B3092C306A44]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.analytics.AbstractAnalyticsStatsTest.getStatResult(AbstractAnalyticsStatsTest.java:113)
	at org.apache.solr.analytics.expression.ExpressionTest.negateTest(ExpressionTest.java:132)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
	at java.lang.reflect.Method.invoke(Method.java:613)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:780)


REGRESSION:  org.apache.solr.analytics.expression.ExpressionTest.powerTest

Error Message:
Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='pr']/double[@name='sum']  raw data: <?xml version="1.0" encoding="UTF-8"?> <response>  <lst name="responseHeader">   <int name="status">0</int>   <int name="QTime">7</int> </lst> <result name="response" numFound="100" start="0"> </result> <lst name="stats">   <lst name="stats_fields"/> </lst> <lst name="stats">   <lst name="nr">     <double name="c">-100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="rr">     <str name="max">str9</str>     <str name="min">str0</str>     <str name="rmax">9rts</str>     <str name="rmin">0rts</str>   </lst>   <lst name="cdr">     <date name="cd1">1800-12-31T23:59:59Z</date>     <date name="cd2">1804-06-30T23:59:59Z</date>     <str name="cs1">1800-12-31T23:59:59Z</str>     <str name="cs2">1804-06-30T23:59:59Z</str>   </lst>   <lst name="dr">     <long name="count">100</long>     <double name="mc">0.2891</double>     <double name="mean">28.91</double>     <double name="su">80.30555555555556</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="mr">     <long name="count">100</long>     <double name="mcm">70829.5</double>     <double name="mean">28.91</double>     <double name="median">24.5</double>     <double name="su">104076.0</double>     <double name="sum">2891.0</double>     <long name="unique">36</long>   </lst>   <lst name="avr">     <double name="c">100.0</double>     <long name="count">100</long>     <null name="s"/>     <null name="sum"/>   </lst>   <lst name="ar">     <long name="count">100</long>     <null name="mcm"/>     <null name="mean"/>     <null name="median"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst>   <lst name="csr">     <str name="cs1">this is the first</str>     <str name="cs2">this is the second</str>     <str name="cs3">this is the third</str>   </lst>   <lst name="cr">     <str name="ccmax">this is the secondstr9</str>     <str name="ccmin">this is the firststr0</str>     <str name="csmax">this is the second</str>     <str name="csmin">this is the first</str>     <str name="max">str9</str>     <str name="min">str0</str>   </lst>   <lst name="dmr">     <str name="cma">+2MONTHS</str>     <str name="cme">+2YEARS</str>     <date name="dmma">1812-02-29T23:59:59Z</date>     <date name="dmme">1807-12-31T23:59:59Z</date>     <date name="max">1811-12-31T23:59:59Z</date>     <date name="median">1805-12-31T23:59:59Z</date>   </lst>   <lst name="cnr">     <double name="c10">10.0</double>     <double name="c8">8.0</double>   </lst>   <lst name="pr">     <long name="count">100</long>     <null name="mc"/>     <null name="mean"/>     <null name="su"/>     <null name="sum"/>     <null name="unique"/>   </lst> </lst> </response> 

Stack Trace:
java.lang.AssertionError: Caught exception in getStatResult, xPath = /response/lst[@name='stats']/lst[@name='pr']/double[@name='sum'] 
raw data: <?xml version="1.0" encoding="UTF-8"?>
<response>

<lst name="responseHeader">
  <int name="status">0</int>
  <int name="QTime">7</int>
</lst>
<result name="response" numFound="100" start="0">
</result>
<lst name="stats">
  <lst name="stats_fields"/>
</lst>
<lst name="stats">
  <lst name="nr">
    <double name="c">-100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="rr">
    <str name="max">str9</str>
    <str name="min">str0</str>
    <str name="rmax">9rts</str>
    <str name="rmin">0rts</str>
  </lst>
  <lst name="cdr">
    <date name="cd1">1800-12-31T23:59:59Z</date>
    <date name="cd2">1804-06-30T23:59:59Z</date>
    <str name="cs1">1800-12-31T23:59:59Z</str>
    <str name="cs2">1804-06-30T23:59:59Z</str>
  </lst>
  <lst name="dr">
    <long name="count">100</long>
    <double name="mc">0.2891</double>
    <double name="mean">28.91</double>
    <double name="su">80.30555555555556</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="mr">
    <long name="count">100</long>
    <double name="mcm">70829.5</double>
    <double name="mean">28.91</double>
    <double name="median">24.5</double>
    <double name="su">104076.0</double>
    <double name="sum">2891.0</double>
    <long name="unique">36</long>
  </lst>
  <lst name="avr">
    <double name="c">100.0</double>
    <long name="count">100</long>
    <null name="s"/>
    <null name="sum"/>
  </lst>
  <lst name="ar">
    <long name="count">100</long>
    <null name="mcm"/>
    <null name="mean"/>
    <null name="median"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
  <lst name="csr">
    <str name="cs1">this is the first</str>
    <str name="cs2">this is the second</str>
    <str name="cs3">this is the third</str>
  </lst>
  <lst name="cr">
    <str name="ccmax">this is the secondstr9</str>
    <str name="ccmin">this is the firststr0</str>
    <str name="csmax">this is the second</str>
    <str name="csmin">this is the first</str>
    <str name="max">str9</str>
    <str name="min">str0</str>
  </lst>
  <lst name="dmr">
    <str name="cma">+2MONTHS</str>
    <str name="cme">+2YEARS</str>
    <date name="dmma">1812-02-29T23:59:59Z</date>
    <date name="dmme">1807-12-31T23:59:59Z</date>
    <date name="max">1811-12-31T23:59:59Z</date>
    <date name="median">1805-12-31T23:59:59Z</date>
  </lst>
  <lst name="cnr">
    <double name="c10">10.0</double>
    <double name="c8">8.0</double>
  </lst>
  <lst name="pr">
    <long name="count">100</long>
    <null name="mc"/>
    <null name="mean"/>
    <null name="su"/>
    <null name="sum"/>
    <null name="unique"/>
  </lst>
</lst>
</response>

	at __randomizedtesting.SeedInfo.seed([9989DBAC881D3AF4:F73B6CAF600A6425]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.analytics.AbstractAnalyticsStatsTest.getStatResult(AbstractAnalyticsStatsTest.java:113)
	at org.apache.solr.analytics.expression.ExpressionTest.powerTest(ExpressionTest.java:119)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
	at java.lang.reflect.Method.invoke(Method.java:613)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:780)


REGRESSION:  org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest.testMultiThreaded

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([9989DBAC881D3AF4:F81C014C60CC3002]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest.checkNumDocs(SignatureUpdateProcessorFactoryTest.java:71)
	at org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest.testMultiThreaded(SignatureUpdateProcessorFactoryTest.java:222)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
	at java.lang.reflect.Method.invoke(Method.java:613)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:780)


REGRESSION:  org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest.testDupeDetection

Error Message:
expected:<1> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<3>
	at __randomizedtesting.SeedInfo.seed([9989DBAC881D3AF4:C2B920648B5A9988]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest.checkNumDocs(SignatureUpdateProcessorFactoryTest.java:71)
	at org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest.testDupeDetection(SignatureUpdateProcessorFactoryTest.java:119)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
	at java.lang.reflect.Method.invoke(Method.java:613)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:780)




Build Log:
[...truncated 10547 lines...]
   [junit4] Suite: org.apache.solr.update.processor.SignatureUpdateProcessorFactoryTest
   [junit4]   2> 494181 T2252 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-SignatureUpdateProcessorFactoryTest-1388779558649
   [junit4]   2> 494181 T2252 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/'
   [junit4]   2> 494182 T2252 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 494182 T2252 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 494214 T2252 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 494255 T2252 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 494255 T2252 oass.IndexSchema.readSchema Reading Solr Schema from schema12.xml
   [junit4]   2> 494260 T2252 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 494568 T2252 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 494570 T2252 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 494572 T2252 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 494577 T2252 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 494580 T2252 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 494583 T2252 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 494584 T2252 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr
   [junit4]   2> 494584 T2252 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/'
   [junit4]   2> 494590 T2252 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 494591 T2252 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr
   [junit4]   2> 494591 T2252 oasc.SolrResourceLoader.<init> new SolrResourceLoader for deduced Solr Home: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/'
   [junit4]   2> 494627 T2252 oasc.CoreContainer.<init> New CoreContainer 852044738
   [junit4]   2> 494628 T2252 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/]
   [junit4]   2> 494628 T2252 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 494629 T2252 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 494629 T2252 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 494629 T2252 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 494630 T2252 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 494630 T2252 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 494630 T2252 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 494631 T2252 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 494631 T2252 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 494634 T2252 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 494634 T2252 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 494634 T2252 oasc.CoreContainer.load Host Name: 
   [junit4]   2> 494638 T2253 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1
   [junit4]   2> 494639 T2253 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/'
   [junit4]   2> 494639 T2253 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 494639 T2253 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 494673 T2253 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 494721 T2253 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 494721 T2253 oass.IndexSchema.readSchema Reading Solr Schema from schema12.xml
   [junit4]   2> 494727 T2253 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 495007 T2253 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 495010 T2253 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 495011 T2253 oass.IndexSchema.readSchema WARN Field text is not multivalued and destination for multiple copyFields (2)
   [junit4]   2> 495016 T2253 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 495019 T2253 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 495022 T2253 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 495022 T2253 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/, dataDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-SignatureUpdateProcessorFactoryTest-1388779558649/
   [junit4]   2> 495022 T2253 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@97a41c8e
   [junit4]   2> 495023 T2253 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-SignatureUpdateProcessorFactoryTest-1388779558649
   [junit4]   2> 495023 T2253 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-SignatureUpdateProcessorFactoryTest-1388779558649/index/
   [junit4]   2> 495024 T2253 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-SignatureUpdateProcessorFactoryTest-1388779558649/index' doesn't exist. Creating new index...
   [junit4]   2> 495024 T2253 oasc.CachingDirectoryFactory.get return new directory for /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-SignatureUpdateProcessorFactoryTest-1388779558649/index
   [junit4]   2> 495024 T2253 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=47.5576171875, floorSegmentMB=1.0390625, forceMergeDeletesPctAllowed=23.44731741168569, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 495025 T2253 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_1,generation=1}
   [junit4]   2> 495025 T2253 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 495025 T2253 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 495027 T2253 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 495028 T2253 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 495028 T2253 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 495028 T2253 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 495028 T2253 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 495029 T2253 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 495029 T2253 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 495029 T2253 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 495030 T2253 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 495030 T2253 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 495030 T2253 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 495030 T2253 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 495031 T2253 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 495031 T2253 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 495031 T2253 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 495032 T2253 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 495032 T2253 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 495032 T2253 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 495033 T2253 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 495033 T2253 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 495033 T2253 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 495034 T2253 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 495034 T2253 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 495034 T2253 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 495035 T2253 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 495035 T2253 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495035 T2253 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495036 T2253 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495036 T2253 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495036 T2253 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495037 T2253 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495037 T2253 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495037 T2253 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 495038 T2253 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 495038 T2253 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 495039 T2253 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 495040 T2253 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 495042 T2253 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 495042 T2253 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 495043 T2253 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6932960683544638]
   [junit4]   2> 495043 T2253 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_1,generation=1}
   [junit4]   2> 495043 T2253 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 495044 T2253 oass.SolrIndexSearcher.<init> Opening Searcher@7b9e5814 main
   [junit4]   2> 495044 T2253 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 495162 T2253 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 495179 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b9e5814 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 495183 T2253 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 495184 T2252 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 495187 T2252 oas.SolrTestCaseJ4.setUp ###Starting testMultiThreaded
   [junit4]   2> ASYNC  NEW_CORE C779 name=collection1 org.apache.solr.core.SolrCore@11dd1e6b
   [junit4]   2> 495192 T2252 C779 oasu.DirectUpdateHandler2.deleteAll [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 495192 T2252 C779 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {deleteByQuery=*:*} 0 0
   [junit4]   2> 495200 T2252 C779 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495201 T2252 C779 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_2,generation=2}
   [junit4]   2> 495201 T2252 C779 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 495202 T2252 C779 oass.SolrIndexSearcher.<init> Opening Searcher@f7bc870e main
   [junit4]   2> 495203 T2252 C779 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495205 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@f7bc870e main{StandardDirectoryReader(segments_2:2:nrt)}
   [junit4]   2> 495206 T2252 C779 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 6
   [junit4]   2> 495219 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495249 T2263 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_3,generation=3}
   [junit4]   2> 495250 T2263 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 495267 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@5e193980 main
   [junit4]   2> 495267 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495268 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495277 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5e193980 main{StandardDirectoryReader(segments_3:18:nrt _4(5.0):C68/67:delGen=1 _3(5.0):C34/31:delGen=1)}
   [junit4]   2> 495289 T2265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_4,generation=4}
   [junit4]   2> 495289 T2265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 495300 T2265 oass.SolrIndexSearcher.<init> Opening Searcher@3234f7e4 main
   [junit4]   2> 495301 T2265 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495301 T2264 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495303 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3234f7e4 main{StandardDirectoryReader(segments_4:32:nrt _7(5.0):C23/19:delGen=1)}
   [junit4]   2> 495310 T2264 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_5,generation=5}
   [junit4]   2> 495311 T2264 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 495317 T2264 oass.SolrIndexSearcher.<init> Opening Searcher@bc72b3e1 main
   [junit4]   2> 495318 T2264 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495319 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495320 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@bc72b3e1 main{StandardDirectoryReader(segments_5:43:nrt _c(5.0):C8/7:delGen=1 _d(5.0):C4/2:delGen=1)}
   [junit4]   2> 495323 T2263 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_5,generation=5}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_6,generation=6}
   [junit4]   2> 495323 T2263 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 6
   [junit4]   2> 495326 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@dd3908fe main
   [junit4]   2> 495326 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495326 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495328 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@dd3908fe main{StandardDirectoryReader(segments_6:50:nrt _e(5.0):C10/8:delGen=1 _f(5.0):C1)}
   [junit4]   2> 495329 T2265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_6,generation=6}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_7,generation=7}
   [junit4]   2> 495330 T2265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 7
   [junit4]   2> 495332 T2265 oass.SolrIndexSearcher.<init> Opening Searcher@9ed1e318 main
   [junit4]   2> 495333 T2265 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495333 T2264 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495334 T2264 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_7,generation=7}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_8,generation=8}
   [junit4]   2> 495334 T2264 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 8
   [junit4]   2> 495335 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9ed1e318 main{StandardDirectoryReader(segments_7:56:nrt _e(5.0):C10/9:delGen=2 _f(5.0):C1 _h(5.0):C1)}
   [junit4]   2> 495335 T2264 oass.SolrIndexSearcher.<init> Opening Searcher@449cb3f5 main
   [junit4]   2> 495335 T2264 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495336 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495336 T2263 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 495337 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@449cb3f5 main{StandardDirectoryReader(segments_7:56:nrt _e(5.0):C10/9:delGen=2 _f(5.0):C1 _h(5.0):C1)}
   [junit4]   2> 495339 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@8cf8f065 main
   [junit4]   2> 495340 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495340 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495342 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8cf8f065 main{StandardDirectoryReader(segments_8:60:nrt _e(5.0):C10/9:delGen=2 _h(5.0):C1 _i(5.0):C1)}
   [junit4]   2> 495343 T2265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_8,generation=8}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_9,generation=9}
   [junit4]   2> 495344 T2265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 9
   [junit4]   2> 495346 T2265 oass.SolrIndexSearcher.<init> Opening Searcher@778901ed main
   [junit4]   2> 495347 T2265 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495347 T2264 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495348 T2264 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_9,generation=9}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_a,generation=10}
   [junit4]   2> 495348 T2264 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 10
   [junit4]   2> 495349 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@778901ed main{StandardDirectoryReader(segments_9:66:nrt _j(5.0):C1 _k(5.0):C1)}
   [junit4]   2> 495349 T2264 oass.SolrIndexSearcher.<init> Opening Searcher@be841d56 main
   [junit4]   2> 495349 T2264 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495350 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495350 T2263 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 495351 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@be841d56 main{StandardDirectoryReader(segments_9:66:nrt _j(5.0):C1 _k(5.0):C1)}
   [junit4]   2> 495353 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@7a348d24 main
   [junit4]   2> 495354 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495354 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495356 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a348d24 main{StandardDirectoryReader(segments_a:71:nrt _j(5.0):C1 _l(5.0):C1)}
   [junit4]   2> 495357 T2265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_a,generation=10}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_b,generation=11}
   [junit4]   2> 495358 T2265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 11
   [junit4]   2> 495360 T2265 oass.SolrIndexSearcher.<init> Opening Searcher@c62893a9 main
   [junit4]   2> 495361 T2265 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495361 T2264 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495362 T2264 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_b,generation=11}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_c,generation=12}
   [junit4]   2> 495362 T2264 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 12
   [junit4]   2> 495363 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c62893a9 main{StandardDirectoryReader(segments_b:78:nrt _j(5.0):C1 _n(5.0):C1)}
   [junit4]   2> 495363 T2264 oass.SolrIndexSearcher.<init> Opening Searcher@d2bb0b56 main
   [junit4]   2> 495364 T2264 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495364 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495364 T2263 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 495365 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@d2bb0b56 main{StandardDirectoryReader(segments_b:78:nrt _j(5.0):C1 _n(5.0):C1)}
   [junit4]   2> 495368 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@10f53a3f main
   [junit4]   2> 495368 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495369 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495370 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@10f53a3f main{StandardDirectoryReader(segments_c:83:nrt _j(5.0):C1 _o(5.0):C1)}
   [junit4]   2> 495371 T2265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_c,generation=12}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_d,generation=13}
   [junit4]   2> 495371 T2265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 13
   [junit4]   2> 495371 T2265 oass.SolrIndexSearcher.<init> Opening Searcher@87eaa520 main
   [junit4]   2> 495372 T2265 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495372 T2264 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495372 T2264 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 495373 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@87eaa520 main{StandardDirectoryReader(segments_d:86:nrt _j(5.0):C1 _p(5.0):C1)}
   [junit4]   2> 495375 T2264 oass.SolrIndexSearcher.<init> Opening Searcher@9547d73e main
   [junit4]   2> 495376 T2264 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495376 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495378 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9547d73e main{StandardDirectoryReader(segments_d:88:nrt _j(5.0):C1 _p(5.0):C1 _q(5.0):C1)}
   [junit4]   2> 495378 T2263 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_d,generation=13}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_e,generation=14}
   [junit4]   2> 495379 T2263 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 14
   [junit4]   2> 495379 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@8069ce76 main
   [junit4]   2> 495380 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495380 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495380 T2265 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 495381 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8069ce76 main{StandardDirectoryReader(segments_e:92:nrt _p(5.0):C1 _r(5.0):C1)}
   [junit4]   2> 495383 T2265 oass.SolrIndexSearcher.<init> Opening Searcher@e0e797c5 main
   [junit4]   2> 495383 T2265 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495384 T2264 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495385 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e0e797c5 main{StandardDirectoryReader(segments_e:97:nrt _p(5.0):C1 _s(5.0):C1)}
   [junit4]   2> 495385 T2264 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_e,generation=14}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_f,generation=15}
   [junit4]   2> 495386 T2264 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 15
   [junit4]   2> 495386 T2264 oass.SolrIndexSearcher.<init> Opening Searcher@62577d62 main
   [junit4]   2> 495387 T2264 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495387 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495388 T2263 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 495388 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@62577d62 main{StandardDirectoryReader(segments_f:98:nrt _p(5.0):C1 _s(5.0):C1 _t(5.0):C1)}
   [junit4]   2> 495390 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@21f6430f main
   [junit4]   2> 495391 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495391 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495392 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@21f6430f main{StandardDirectoryReader(segments_f:103:nrt _p(5.0):C1 _s(5.0):C1 _u(5.0):C1)}
   [junit4]   2> 495393 T2265 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_f,generation=15}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_g,generation=16}
   [junit4]   2> 495393 T2265 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 16
   [junit4]   2> 495394 T2265 oass.SolrIndexSearcher.<init> Opening Searcher@586dcd6e main
   [junit4]   2> 495394 T2265 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495395 T2264 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495395 T2264 oasu.DirectUpdateHandler2.commit No uncommitted changes. Skipping IW.commit.
   [junit4]   2> 495396 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@586dcd6e main{StandardDirectoryReader(segments_g:106:nrt _p(5.0):C1 _s(5.0):C1 _v(5.0):C1)}
   [junit4]   2> 495398 T2264 oass.SolrIndexSearcher.<init> Opening Searcher@2a8f40ba main
   [junit4]   2> 495398 T2264 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495399 T2263 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 495400 T2254 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2a8f40ba main{StandardDirectoryReader(segments_g:108:nrt _p(5.0):C1 _s(5.0):C1 _v(5.0):C1 _w(5.0):C1)}
   [junit4]   2> 495402 T2263 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_g,generation=16}
   [junit4]   2> 		commit{dir=RateLimitedDirectoryWrapper(MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@432473bd lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@55f2e6c1)),segFN=segments_h,generation=17}
   [junit4]   2> 495402 T2263 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 17
   [junit4]   2> 495404 T2263 oass.SolrIndexSearcher.<init> Opening Searcher@c93dddd9 main
   [junit4]   2> 495404 T2263 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 495405 T2265 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 4

[...truncated too long message...]

nique"/>
   [junit4]    >   </lst>
   [junit4]    >   <lst name="csr">
   [junit4]    >     <str name="cs1">this is the first</str>
   [junit4]    >     <str name="cs2">this is the second</str>
   [junit4]    >     <str name="cs3">this is the third</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst name="cr">
   [junit4]    >     <str name="ccmax">this is the secondstr9</str>
   [junit4]    >     <str name="ccmin">this is the firststr0</str>
   [junit4]    >     <str name="csmax">this is the second</str>
   [junit4]    >     <str name="csmin">this is the first</str>
   [junit4]    >     <str name="max">str9</str>
   [junit4]    >     <str name="min">str0</str>
   [junit4]    >   </lst>
   [junit4]    >   <lst name="dmr">
   [junit4]    >     <str name="cma">+2MONTHS</str>
   [junit4]    >     <str name="cme">+2YEARS</str>
   [junit4]    >     <date name="dmma">1812-02-29T23:59:59Z</date>
   [junit4]    >     <date name="dmme">1807-12-31T23:59:59Z</date>
   [junit4]    >     <date name="max">1811-12-31T23:59:59Z</date>
   [junit4]    >     <date name="median">1805-12-31T23:59:59Z</date>
   [junit4]    >   </lst>
   [junit4]    >   <lst name="cnr">
   [junit4]    >     <double name="c10">10.0</double>
   [junit4]    >     <double name="c8">8.0</double>
   [junit4]    >   </lst>
   [junit4]    >   <lst name="pr">
   [junit4]    >     <long name="count">100</long>
   [junit4]    >     <null name="mc"/>
   [junit4]    >     <null name="mean"/>
   [junit4]    >     <null name="su"/>
   [junit4]    >     <null name="sum"/>
   [junit4]    >     <null name="unique"/>
   [junit4]    >   </lst>
   [junit4]    > </lst>
   [junit4]    > </response>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9989DBAC881D3AF4:F73B6CAF600A6425]:0)
   [junit4]    > 	at org.apache.solr.analytics.AbstractAnalyticsStatsTest.getStatResult(AbstractAnalyticsStatsTest.java:113)
   [junit4]    > 	at org.apache.solr.analytics.expression.ExpressionTest.powerTest(ExpressionTest.java:119)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:780)
   [junit4]   2> 550623 T2556 oas.SolrTestCaseJ4.setUp ###Starting multiplyTest
   [junit4]   2> 550629 T2556 oas.SolrTestCaseJ4.tearDown ###Ending multiplyTest
   [junit4]   2> 550630 T2556 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 550631 T2556 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=-1076300929
   [junit4]   2> 550631 T2556 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@d9b9b7be
   [junit4]   2> 550632 T2556 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=100,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 550632 T2556 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 550632 T2556 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 550633 T2556 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 550633 T2556 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 550634 T2556 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 550634 T2556 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-ExpressionTest-1388779614210 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-ExpressionTest-1388779614210;done=false>>]
   [junit4]   2> 550634 T2556 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-ExpressionTest-1388779614210
   [junit4]   2> 550635 T2556 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-ExpressionTest-1388779614210/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-ExpressionTest-1388779614210/index;done=false>>]
   [junit4]   2> 550635 T2556 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-ExpressionTest-1388779614210/index
   [junit4]   2> NOTE: test params are: codec=Lucene45, sim=DefaultSimilarity, locale=in, timezone=Asia/Rangoon
   [junit4]   2> NOTE: Linux 3.8.0-34-generic amd64/IBM Corporation 1.7.0 (64-bit)/cpus=8,threads=1,free=111471464,total=271515648
   [junit4]   2> NOTE: All tests run in this JVM: [ShowFileRequestHandlerTest, FieldMutatingUpdateProcessorTest, DOMUtilTest, FastVectorHighlighterTest, TestDistributedMissingSort, BlockDirectoryTest, BlockCacheTest, TestLMJelinekMercerSimilarityFactory, TestManagedSchema, TestJmxMonitoredMap, TestNRTOpen, QueryEqualityTest, TestCharFilters, TermsComponentTest, TestPerFieldSimilarity, TestCloudManagedSchema, LeaderElectionTest, TestSolrQueryParserResource, TestRandomDVFaceting, TestCloudManagedSchemaAddField, InfoHandlerTest, TestCustomSort, AutoCommitTest, SoftAutoCommitTest, TestMergePolicyConfig, TestLMDirichletSimilarityFactory, TestDefaultSearchFieldResource, DistributedSuggestComponentTest, TestSuggestSpellingConverter, TestModifyConfFiles, TestSolrIndexConfig, ZkControllerTest, SampleTest, TestReplicationHandler, TestLRUCache, TestFieldResource, TestReversedWildcardFilterFactory, ClusterStateTest, SimpleFacetsTest, TestSolrDeletionPolicy1, TestNoOpRegenerator, TestBM25SimilarityFactory, UUIDFieldTest, SyncSliceTest, ResponseLogComponentTest, BJQParserTest, MoreLikeThisHandlerTest, SearchHandlerTest, TestRemoteStreaming, TestInfoStreamLogging, TestSort, HdfsLockFactoryTest, ZkCLITest, DeleteReplicaTest, TestDocumentBuilder, UpdateRequestProcessorFactoryTest, ReturnFieldsTest, TestSerializedLuceneMatchVersion, DocValuesTest, QueryElevationComponentTest, PrimitiveFieldTypeTest, TestManagedSchemaFieldResource, TestSolrXMLSerializer, TestRandomMergePolicy, CopyFieldTest, DocumentBuilderTest, StressHdfsTest, TestAnalyzeInfixSuggestions, EchoParamsTest, BadComponentTest, TestSystemIdResolver, RecoveryZkTest, NoFacetTest, SchemaVersionSpecificBehaviorTest, SuggesterTest, ScriptEngineTest, TestAddFieldRealTimeGet, JsonLoaderTest, DistributedSpellCheckComponentTest, TestSchemaResource, DistributedTermsComponentTest, TestSurroundQueryParser, URLClassifyProcessorTest, TestJoin, TestValueSourceCache, ShardRoutingTest, BasicZkTest, TestDistributedSearch, TestDistributedGrouping, TestFaceting, TestHashPartitioner, TestStressReorder, TestRangeQuery, TestCoreContainer, SolrCoreTest, PeerSyncTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFiltering, OverseerCollectionProcessorTest, DistributedQueryElevationComponentTest, SolrIndexSplitterTest, SignatureUpdateProcessorFactoryTest, SpellCheckCollatorTest, SuggesterTSTTest, TestStressLucene, TestTrie, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, StatelessScriptUpdateProcessorFactoryTest, SolrInfoMBeanTest, DisMaxRequestHandlerTest, TestQueryUtils, TestOmitPositions, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, FieldAnalysisRequestHandlerTest, TestArbitraryIndexDir, RegexBoostProcessorTest, MBeansHandlerTest, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestBinaryResponseWriter, SOLR749Test, AlternateDirectoryTest, SolrIndexConfigTest, MultiTermTest, TestDocSet, TestSearchPerf, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, SpellPossibilityIteratorTest, SynonymTokenizerTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestNumberUtils, LegacyHTMLStripCharFilterTest, ResourceLoaderTest, TestFastOutputStream, PluginInfoTest, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, PrimUtilsTest, DateFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestSolrJ, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, CircularListTest, TestRTGBase, SolrTestCaseJ4Test, ExpressionTest]
   [junit4] Completed on J0 in 0.92s, 12 tests, 4 failures <<< FAILURES!

[...truncated 156 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:453: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:446: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:491: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1304: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:937: There were test failures: 358 suites, 1577 tests, 6 failures, 67 ignored (6 assumptions)

Total time: 35 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/ibm-j9-jdk7 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-trunk-Linux (64bit/jdk1.8.0-ea-b121) - Build # 8944 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/8944/
Java: 64bit/jdk1.8.0-ea-b121 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MigrateRouteKeyTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.MigrateRouteKeyTest:     1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.MigrateRouteKeyTest: 
   1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
	at __randomizedtesting.SeedInfo.seed([83244FFE893D3F26]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MigrateRouteKeyTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at java.util.TimerThread.mainLoop(Timer.java:526)         at java.util.TimerThread.run(Timer.java:505)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at java.util.TimerThread.mainLoop(Timer.java:526)
        at java.util.TimerThread.run(Timer.java:505)
	at __randomizedtesting.SeedInfo.seed([83244FFE893D3F26]:0)




Build Log:
[...truncated 10881 lines...]
   [junit4] Suite: org.apache.solr.cloud.MigrateRouteKeyTest
   [junit4]   2> 928395 T5385 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ad_ud/um
   [junit4]   2> 928398 T5385 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-MigrateRouteKeyTest-1388774771528
   [junit4]   2> 928398 T5385 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 928399 T5386 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 928499 T5385 oasc.ZkTestServer.run start zk server on port:51732
   [junit4]   2> 928500 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928671 T5393 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2eeb686a name:ZooKeeperConnection Watcher:127.0.0.1:51732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928671 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928672 T5385 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 928676 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928677 T5396 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c1d48d2 name:ZooKeeperConnection Watcher:127.0.0.1:51732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928678 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928678 T5385 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 928681 T5385 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 928682 T5385 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 928684 T5385 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 928685 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 928685 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 928688 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 928688 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 928690 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 928690 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 928692 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 928692 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 928694 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 928694 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 928696 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 928696 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 928697 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 928698 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 928699 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 928699 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 928701 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 928701 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 928702 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 928702 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 928704 T5385 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 928704 T5385 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 928768 T5385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 928774 T5385 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54791
   [junit4]   2> 928774 T5385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 928775 T5385 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 928775 T5385 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836
   [junit4]   2> 928775 T5385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/'
   [junit4]   2> 928785 T5385 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/solr.xml
   [junit4]   2> 928812 T5385 oasc.CoreContainer.<init> New CoreContainer 1970165306
   [junit4]   2> 928812 T5385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/]
   [junit4]   2> 928813 T5385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 928813 T5385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 928813 T5385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 928813 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 928814 T5385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 928814 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 928814 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 928814 T5385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 928815 T5385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 928816 T5385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 928817 T5385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 928817 T5385 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 928817 T5385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51732/solr
   [junit4]   2> 928817 T5385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 928818 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928820 T5408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57566315 name:ZooKeeperConnection Watcher:127.0.0.1:51732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928821 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928823 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 928824 T5411 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62d93f56 name:ZooKeeperConnection Watcher:127.0.0.1:51732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 928824 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 928825 T5385 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 928827 T5385 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 928829 T5385 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 928830 T5385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54791_ad_ud%2Fum
   [junit4]   2> 928831 T5385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54791_ad_ud%2Fum
   [junit4]   2> 928832 T5385 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 928834 T5385 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 928836 T5385 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 928838 T5385 oasc.Overseer.start Overseer (id=91014743426990083-127.0.0.1:54791_ad_ud%2Fum-n_0000000000) starting
   [junit4]   2> 928840 T5385 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 928843 T5413 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 928843 T5385 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 928844 T5385 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 928846 T5385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 928847 T5412 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 928849 T5414 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 928849 T5414 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 928854 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928854 T5414 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 928856 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 928856 T5412 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54791/ad_ud/um",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54791_ad_ud%2Fum",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 928856 T5412 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 928857 T5412 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 928861 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 928861 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 929855 T5414 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 929855 T5414 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/collection1
   [junit4]   2> 929855 T5414 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 929856 T5414 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 929856 T5414 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 929857 T5414 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/collection1/'
   [junit4]   2> 929857 T5414 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/collection1/lib/classes/' to classloader
   [junit4]   2> 929858 T5414 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/collection1/lib/README' to classloader
   [junit4]   2> 929872 T5414 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 929880 T5414 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 929881 T5414 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 929885 T5414 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 929974 T5414 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 929976 T5414 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 929976 T5414 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 929978 T5414 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 929991 T5414 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 929992 T5414 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.MigrateRouteKeyTest-controljetty-1388774771836/collection1/, dataDir=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/control/data/
   [junit4]   2> 929992 T5414 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e374d04
   [junit4]   2> 929993 T5414 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/control/data
   [junit4]   2> 929993 T5414 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/control/data/index/
   [junit4]   2> 929993 T5414 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 929993 T5414 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/control/data/index
   [junit4]   2> 929994 T5414 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 929995 T5414 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7640c013 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5a19b679),segFN=segments_1,generation=1}
   [junit4]   2> 929995 T5414 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 929997 T5414 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 929998 T5414 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 929998 T5414 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 929998 T5414 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 929998 T5414 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 929999 T5414 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 929999 T5414 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 929999 T5414 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 930000 T5414 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 930000 T5414 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 930000 T5414 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 930000 T5414 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 930001 T5414 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 930001 T5414 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 930001 T5414 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 930002 T5414 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 930002 T5414 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 930004 T5414 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 930006 T5414 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 930006 T5414 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 930007 T5414 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43061104124444793]
   [junit4]   2> 930007 T5414 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7640c013 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5a19b679),segFN=segments_1,generation=1}
   [junit4]   2> 930008 T5414 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 930008 T5414 oass.SolrIndexSearcher.<init> Opening Searcher@ba0a8b5 main
   [junit4]   2> 930010 T5415 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ba0a8b5 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 930010 T5414 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 930011 T5385 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 930011 T5385 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 930011 T5418 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54791/ad_ud/um collection:control_collection shard:shard1
   [junit4]   2> 930013 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930013 T5418 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 930014 T5421 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b216962 name:ZooKeeperConnection Watcher:127.0.0.1:51732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930014 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930015 T5385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 930018 T5418 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 930018 T5385 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 930022 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930023 T5418 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 930023 T5418 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3952 name=collection1 org.apache.solr.core.SolrCore@79e46725 url=http://127.0.0.1:54791/ad_ud/um/collection1 node=127.0.0.1:54791_ad_ud%2Fum C3952_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:54791/ad_ud/um, core=collection1, node_name=127.0.0.1:54791_ad_ud%2Fum}
   [junit4]   2> 930023 T5418 C3952 P54791 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54791/ad_ud/um/collection1/
   [junit4]   2> 930023 T5418 C3952 P54791 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 930023 T5418 C3952 P54791 oasc.SyncStrategy.syncToMe http://127.0.0.1:54791/ad_ud/um/collection1/ has no replicas
   [junit4]   2> 930023 T5418 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54791/ad_ud/um/collection1/ shard1
   [junit4]   2> 930024 T5418 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 930024 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 930027 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930032 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930034 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930091 T5385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 930093 T5385 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:38102
   [junit4]   2> 930093 T5385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 930093 T5385 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 930094 T5385 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148
   [junit4]   2> 930094 T5385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/'
   [junit4]   2> 930104 T5385 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/solr.xml
   [junit4]   2> 930112 T5385 oasc.CoreContainer.<init> New CoreContainer 159441439
   [junit4]   2> 930112 T5385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/]
   [junit4]   2> 930112 T5385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 930113 T5385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 930113 T5385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 930113 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 930113 T5385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 930113 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 930114 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 930114 T5385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 930114 T5385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 930116 T5385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 930116 T5385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 930116 T5385 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 930116 T5385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51732/solr
   [junit4]   2> 930116 T5385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 930118 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930121 T5433 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44062c32 name:ZooKeeperConnection Watcher:127.0.0.1:51732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930121 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930124 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 930126 T5436 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@54357957 name:ZooKeeperConnection Watcher:127.0.0.1:51732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 930126 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 930133 T5385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 930136 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930136 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930136 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930182 T5418 oasc.ZkController.register We are http://127.0.0.1:54791/ad_ud/um/collection1/ and leader is http://127.0.0.1:54791/ad_ud/um/collection1/
   [junit4]   2> 930182 T5418 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54791/ad_ud/um
   [junit4]   2> 930183 T5418 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 930183 T5418 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 930183 T5418 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 930184 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930184 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930184 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930184 T5418 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 930186 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 930187 T5412 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54791/ad_ud/um",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54791_ad_ud%2Fum",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 930190 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 930292 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930292 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 930292 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 931137 T5385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38102_ad_ud%2Fum
   [junit4]   2> 931138 T5385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38102_ad_ud%2Fum
   [junit4]   2> 931140 T5421 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 931140 T5411 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 931140 T5436 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 931143 T5437 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 931143 T5437 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 931143 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931143 T5437 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 931144 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931144 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931145 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 931145 T5412 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38102/ad_ud/um",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38102_ad_ud%2Fum",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 931145 T5412 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 931145 T5412 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 931147 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 931250 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931250 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 931250 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932144 T5437 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 932144 T5437 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/collection1
   [junit4]   2> 932145 T5437 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 932146 T5437 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 932146 T5437 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 932147 T5437 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/collection1/'
   [junit4]   2> 932148 T5437 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/collection1/lib/classes/' to classloader
   [junit4]   2> 932149 T5437 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/collection1/lib/README' to classloader
   [junit4]   2> 932164 T5437 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 932177 T5437 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 932178 T5437 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 932182 T5437 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 932275 T5437 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 932276 T5437 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 932277 T5437 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 932280 T5437 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 932293 T5437 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 932294 T5437 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.MigrateRouteKeyTest-jetty1-1388774773148/collection1/, dataDir=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty1/
   [junit4]   2> 932294 T5437 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e374d04
   [junit4]   2> 932294 T5437 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty1
   [junit4]   2> 932295 T5437 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty1/index/
   [junit4]   2> 932295 T5437 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 932295 T5437 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty1/index
   [junit4]   2> 932296 T5437 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 932296 T5437 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@71b8c524 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3cf7288),segFN=segments_1,generation=1}
   [junit4]   2> 932296 T5437 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 932304 T5437 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 932304 T5437 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 932305 T5437 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 932305 T5437 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 932305 T5437 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 932305 T5437 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 932306 T5437 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 932306 T5437 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 932306 T5437 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 932307 T5437 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 932307 T5437 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 932307 T5437 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 932307 T5437 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 932308 T5437 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 932308 T5437 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 932309 T5437 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 932309 T5437 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 932311 T5437 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 932313 T5437 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 932313 T5437 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 932313 T5437 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43061104124444793]
   [junit4]   2> 932314 T5437 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@71b8c524 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3cf7288),segFN=segments_1,generation=1}
   [junit4]   2> 932314 T5437 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 932314 T5437 oass.SolrIndexSearcher.<init> Opening Searcher@7a437c7a main
   [junit4]   2> 932316 T5438 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7a437c7a main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 932318 T5437 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 932319 T5385 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 932319 T5385 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 932319 T5441 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38102/ad_ud/um collection:collection1 shard:shard2
   [junit4]   2> 932320 T5441 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 932326 T5441 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 932327 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932327 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932327 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932327 T5441 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 932327 T5441 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3953 name=collection1 org.apache.solr.core.SolrCore@2f6608c9 url=http://127.0.0.1:38102/ad_ud/um/collection1 node=127.0.0.1:38102_ad_ud%2Fum C3953_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:38102/ad_ud/um, core=collection1, node_name=127.0.0.1:38102_ad_ud%2Fum}
   [junit4]   2> 932328 T5441 C3953 P38102 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38102/ad_ud/um/collection1/
   [junit4]   2> 932328 T5441 C3953 P38102 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 932328 T5441 C3953 P38102 oasc.SyncStrategy.syncToMe http://127.0.0.1:38102/ad_ud/um/collection1/ has no replicas
   [junit4]   2> 932328 T5441 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38102/ad_ud/um/collection1/ shard2
   [junit4]   2> 932328 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 932328 T5441 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 932330 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932332 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932333 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932390 T5385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 932392 T5385 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:45361
   [junit4]   2> 932393 T5385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 932393 T5385 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 932393 T5385 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449
   [junit4]   2> 932393 T5385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/'
   [junit4]   2> 932402 T5385 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/solr.xml
   [junit4]   2> 932410 T5385 oasc.CoreContainer.<init> New CoreContainer 1122965322
   [junit4]   2> 932411 T5385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/]
   [junit4]   2> 932411 T5385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 932411 T5385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 932411 T5385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 932412 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 932412 T5385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 932412 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 932412 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 932413 T5385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 932413 T5385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 932414 T5385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 932415 T5385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 932415 T5385 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 932415 T5385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51732/solr
   [junit4]   2> 932416 T5385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 932416 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 932418 T5453 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@aafb6bf name:ZooKeeperConnection Watcher:127.0.0.1:51732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 932418 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 932420 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 932421 T5456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ef3a940 name:ZooKeeperConnection Watcher:127.0.0.1:51732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 932421 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 932423 T5385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 932435 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932436 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932436 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932436 T5456 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932482 T5441 oasc.ZkController.register We are http://127.0.0.1:38102/ad_ud/um/collection1/ and leader is http://127.0.0.1:38102/ad_ud/um/collection1/
   [junit4]   2> 932483 T5441 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38102/ad_ud/um
   [junit4]   2> 932483 T5441 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 932483 T5441 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 932483 T5441 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 932484 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932485 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932485 T5441 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 932485 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932486 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 932486 T5412 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38102/ad_ud/um",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38102_ad_ud%2Fum",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 932488 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 932590 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932590 T5456 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932590 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 932590 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 933431 T5385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45361_ad_ud%2Fum
   [junit4]   2> 933432 T5385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45361_ad_ud%2Fum
   [junit4]   2> 933435 T5411 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 933435 T5421 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 933436 T5456 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 933445 T5436 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 933452 T5457 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 933452 T5457 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 933453 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933453 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933453 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933460 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 933460 T5457 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 933461 T5412 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45361/ad_ud/um",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45361_ad_ud%2Fum",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 933461 T5412 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 933461 T5412 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 933463 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 933565 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933565 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933565 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 933566 T5456 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934461 T5457 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 934461 T5457 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/collection1
   [junit4]   2> 934461 T5457 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 934462 T5457 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 934462 T5457 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 934462 T5457 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/collection1/'
   [junit4]   2> 934463 T5457 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/collection1/lib/classes/' to classloader
   [junit4]   2> 934463 T5457 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/collection1/lib/README' to classloader
   [junit4]   2> 934484 T5457 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 934494 T5457 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 934495 T5457 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 934500 T5457 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 934608 T5457 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 934609 T5457 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 934610 T5457 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 934612 T5457 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 934623 T5457 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 934623 T5457 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty2-1388774775449/collection1/, dataDir=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty2/
   [junit4]   2> 934623 T5457 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e374d04
   [junit4]   2> 934624 T5457 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty2
   [junit4]   2> 934624 T5457 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty2/index/
   [junit4]   2> 934624 T5457 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 934624 T5457 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty2/index
   [junit4]   2> 934625 T5457 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 934625 T5457 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32d5b2c4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ecbd3ff),segFN=segments_1,generation=1}
   [junit4]   2> 934625 T5457 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 934627 T5457 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 934627 T5457 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 934627 T5457 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 934628 T5457 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 934628 T5457 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 934628 T5457 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 934628 T5457 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 934628 T5457 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 934628 T5457 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 934629 T5457 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 934629 T5457 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 934629 T5457 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 934629 T5457 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 934630 T5457 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 934630 T5457 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 934630 T5457 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 934630 T5457 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 934632 T5457 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 934633 T5457 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 934633 T5457 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 934633 T5457 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43061104124444793]
   [junit4]   2> 934634 T5457 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@32d5b2c4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ecbd3ff),segFN=segments_1,generation=1}
   [junit4]   2> 934634 T5457 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 934634 T5457 oass.SolrIndexSearcher.<init> Opening Searcher@7863084b main
   [junit4]   2> 934636 T5458 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7863084b main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 934637 T5457 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 934638 T5385 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 934638 T5385 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 934638 T5461 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45361/ad_ud/um collection:collection1 shard:shard1
   [junit4]   2> 934639 T5461 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 934642 T5461 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 934643 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934644 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934644 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934644 T5461 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 934644 T5461 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3954 name=collection1 org.apache.solr.core.SolrCore@7cd27eca url=http://127.0.0.1:45361/ad_ud/um/collection1 node=127.0.0.1:45361_ad_ud%2Fum C3954_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:45361/ad_ud/um, core=collection1, node_name=127.0.0.1:45361_ad_ud%2Fum}
   [junit4]   2> 934644 T5461 C3954 P45361 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45361/ad_ud/um/collection1/
   [junit4]   2> 934644 T5461 C3954 P45361 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 934644 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 934644 T5461 C3954 P45361 oasc.SyncStrategy.syncToMe http://127.0.0.1:45361/ad_ud/um/collection1/ has no replicas
   [junit4]   2> 934645 T5461 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45361/ad_ud/um/collection1/ shard1
   [junit4]   2> 934645 T5461 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 934648 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934649 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934709 T5385 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 934710 T5385 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:58047
   [junit4]   2> 934711 T5385 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 934711 T5385 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 934711 T5385 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769
   [junit4]   2> 934711 T5385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/'
   [junit4]   2> 934720 T5385 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/solr.xml
   [junit4]   2> 934732 T5385 oasc.CoreContainer.<init> New CoreContainer 1534441170
   [junit4]   2> 934732 T5385 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/]
   [junit4]   2> 934733 T5385 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 934733 T5385 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 934733 T5385 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 934733 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 934734 T5385 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 934734 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 934734 T5385 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 934734 T5385 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 934734 T5385 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 934736 T5385 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 934736 T5385 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 934737 T5385 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 934737 T5385 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51732/solr
   [junit4]   2> 934737 T5385 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 934738 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 934740 T5473 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1995583e name:ZooKeeperConnection Watcher:127.0.0.1:51732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 934740 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 934742 T5385 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 934743 T5476 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@604877a1 name:ZooKeeperConnection Watcher:127.0.0.1:51732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 934743 T5385 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 934745 T5385 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 934751 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934751 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934751 T5476 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934751 T5456 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934751 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934798 T5461 oasc.ZkController.register We are http://127.0.0.1:45361/ad_ud/um/collection1/ and leader is http://127.0.0.1:45361/ad_ud/um/collection1/
   [junit4]   2> 934799 T5461 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45361/ad_ud/um
   [junit4]   2> 934799 T5461 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 934799 T5461 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 934799 T5461 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 934800 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934800 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934800 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934800 T5461 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 934801 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 934801 T5412 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45361/ad_ud/um",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45361_ad_ud%2Fum",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 934803 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 934905 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934905 T5456 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934905 T5476 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934905 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 934905 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 935747 T5385 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58047_ad_ud%2Fum
   [junit4]   2> 935748 T5385 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58047_ad_ud%2Fum
   [junit4]   2> 935750 T5436 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 935750 T5476 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 935750 T5411 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 935750 T5421 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 935750 T5456 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 935753 T5477 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 935753 T5477 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 935754 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935754 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935754 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935754 T5477 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 935755 T5412 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 935755 T5412 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58047/ad_ud/um",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:58047_ad_ud%2Fum",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 935755 T5412 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 935756 T5412 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 935757 T5411 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 935859 T5436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 935859 T5421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 935859 T5456 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 935859 T5476 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 935859 T5411 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 936755 T5477 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 936755 T5477 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/collection1
   [junit4]   2> 936755 T5477 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 936756 T5477 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 936756 T5477 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 936757 T5477 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/collection1/'
   [junit4]   2> 936758 T5477 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/collection1/lib/classes/' to classloader
   [junit4]   2> 936758 T5477 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/collection1/lib/README' to classloader
   [junit4]   2> 936778 T5477 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 936788 T5477 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 936790 T5477 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 936793 T5477 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 936886 T5477 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 936887 T5477 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 936888 T5477 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 936890 T5477 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 936902 T5477 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 936903 T5477 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.MigrateRouteKeyTest-jetty3-1388774777769/collection1/, dataDir=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty3/
   [junit4]   2> 936903 T5477 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e374d04
   [junit4]   2> 936903 T5477 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty3
   [junit4]   2> 936903 T5477 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty3/index/
   [junit4]   2> 936903 T5477 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 936904 T5477 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty3/index
   [junit4]   2> 936904 T5477 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 936904 T5477 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@17df371b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@23f427e0),segFN=segments_1,generation=1}
   [junit4]   2> 936904 T5477 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 936906 T5477 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 936907 T5477 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 936907 T5477 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 936907 T5477 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 936907 T5477 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 936907 T5477 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 936907 T5477 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 936908 T5477 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 936908 T5477 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 936908 T5477 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 936908 T5477 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 936908 T5477 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 936909 T5477 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 936909 T5477 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 936909 T5477 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 936909 T5477 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 936910 T5477 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 936911 T5477 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 936912 T5477 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 936912 T5477 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 936913 T5477 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=18, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.43061104124444793]
   [junit4]   2> 936913 T5477 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@17df371b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@23f427e0),segFN=segments_1,generation=1}
   [junit4]   2> 936913 T5477 oasc.SolrDeletio

[...truncated too long message...]

e looking to close ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty4/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty4/index;done=false>>]
   [junit4]   2> 974043 T5497 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.MigrateRouteKeyTest-1388774771528/jetty4/index
   [junit4]   2> 974043 T5497 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 974044 T5497 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:93)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:138)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 46639 T5384 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 66716 T5384 ccr.ThreadLeakControl.checkThreadLeaks SEVERE 1 thread leaked from SUITE scope at org.apache.solr.cloud.MigrateRouteKeyTest: 
   [junit4]   2> 	   1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.lang.Object.wait(Object.java:502)
   [junit4]   2> 	        at java.util.TimerThread.mainLoop(Timer.java:526)
   [junit4]   2> 	        at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> 66717 T5384 ccr.ThreadLeakControl.tryToInterruptAll Starting to interrupt leaked threads:
   [junit4]   2> 	   1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]
   [junit4]   2> 69719 T5384 ccr.ThreadLeakControl.tryToInterruptAll SEVERE There are still zombie threads that couldn't be terminated:
   [junit4]   2> 	   1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]
   [junit4]   2> 	        at java.lang.Object.wait(Native Method)
   [junit4]   2> 	        at java.lang.Object.wait(Object.java:502)
   [junit4]   2> 	        at java.util.TimerThread.mainLoop(Timer.java:526)
   [junit4]   2> 	        at java.util.TimerThread.run(Timer.java:505)
   [junit4]   2> NOTE: test params are: codec=Lucene46: {_version_=PostingsFormat(name=Direct), n_ti=PostingsFormat(name=Lucene41WithOrds), id=FSTOrd41}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=sv_SE, timezone=CET
   [junit4]   2> NOTE: Linux 3.8.0-34-generic amd64/Oracle Corporation 1.8.0-ea (64-bit)/cpus=8,threads=2,free=269368376,total=405020672
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParserResource, SchemaVersionSpecificBehaviorTest, TestLRUCache, AutoCommitTest, TestModifyConfFiles, FieldMutatingUpdateProcessorTest, TestSolrXmlPersistor, DocValuesTest, FileBasedSpellCheckerTest, UpdateParamsTest, QueryParsingTest, TestRandomDVFaceting, DistributedTermsComponentTest, LukeRequestHandlerTest, TestSearchPerf, SuggesterTest, DirectUpdateHandlerOptimizeTest, TermsComponentTest, AbstractAnalyticsFacetTest, HdfsSyncSliceTest, DateMathParserTest, TestManagedSchemaFieldResource, TestPerFieldSimilarity, UUIDFieldTest, TestLMDirichletSimilarityFactory, QueryEqualityTest, BasicDistributedZkTest, WordBreakSolrSpellCheckerTest, SimpleFacetsTest, TriLevelCompositeIdRoutingTest, TestNoOpRegenerator, TestBM25SimilarityFactory, SystemInfoHandlerTest, TestClassNameShortening, TestSolrDeletionPolicy1, DeleteShardTest, TestMergePolicyConfig, OverseerTest, BadComponentTest, LegacyHTMLStripCharFilterTest, TestCharFilters, HdfsDirectoryTest, TestDocSet, TestLMJelinekMercerSimilarityFactory, TestHighFrequencyDictionaryFactory, SpatialFilterTest, TestSchemaNameResource, TestSolrXMLSerializer, TestRecoveryHdfs, RecoveryZkTest, TestPostingsSolrHighlighter, URLClassifyProcessorTest, TestQueryUtils, TestMaxScoreQueryParser, TestSurroundQueryParser, RegexBoostProcessorTest, DistributedQueryElevationComponentTest, ClusterStateTest, TestConfig, TestSystemIdResolver, ExternalFileFieldSortTest, TestJmxMonitoredMap, SoftAutoCommitTest, TestLuceneMatchVersion, TestSchemaSimilarityResource, SolrCoreTest, EnumFieldTest, BinaryUpdateRequestHandlerTest, StatelessScriptUpdateProcessorFactoryTest, DistributedSuggestComponentTest, ChangedSchemaMergeTest, LoggingHandlerTest, MBeansHandlerTest, DistributedSpellCheckComponentTest, TestRemoteStreaming, CoreMergeIndexesAdminHandlerTest, SpellPossibilityIteratorTest, TestFieldTypeCollectionResource, SliceStateUpdateTest, FastVectorHighlighterTest, AbstractAnalyticsStatsTest, TestSchemaResource, PolyFieldTest, CachingDirectoryFactoryTest, CopyFieldTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, ShardRoutingTest, BasicZkTest, TestRandomFaceting, TestFaceting, ZkControllerTest, TestStressReorder, TestJoin, SolrCmdDistributorTest, QueryElevationComponentTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestFunctionQuery, BasicFunctionalityTest, TestBadConfig, TestIndexSearcher, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, CoreAdminHandlerTest, TestTrie, TestPseudoReturnFields, TestWordDelimiterFilterFactory, SolrInfoMBeanTest, IndexBasedSpellCheckerTest, TestWriterPerf, DocumentAnalysisRequestHandlerTest, RequestHandlersTest, TestValueSourceCache, MoreLikeThisHandlerTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, ReturnFieldsTest, UniqFieldsUpdateProcessorFactoryTest, TestPHPSerializedResponseWriter, PingRequestHandlerTest, TestComponentsName, CoreContainerCoreInitFailuresTest, HighlighterConfigTest, SOLR749Test, ResponseLogComponentTest, SampleTest, TestElisionMultitermQuery, DirectSolrConnectionTest, SynonymTokenizerTest, EchoParamsTest, TestNumberUtils, TestDefaultSimilarityFactory, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, PluginInfoTest, TestSuggestSpellingConverter, DOMUtilTest, TestDocumentBuilder, SliceStateTest, CircularListTest, TestRTGBase, TestDistributedMissingSort, MigrateRouteKeyTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=MigrateRouteKeyTest -Dtests.seed=83244FFE893D3F26 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sv_SE -Dtests.timezone=CET -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | MigrateRouteKeyTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.MigrateRouteKeyTest: 
   [junit4]    >    1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:502)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:526)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([83244FFE893D3F26]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=5539, name=Timer-370, state=WAITING, group=TGRP-MigrateRouteKeyTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.lang.Object.wait(Object.java:502)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:526)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([83244FFE893D3F26]:0)
   [junit4] Completed on J0 in 69.76s, 1 test, 2 errors <<< FAILURES!

[...truncated 150 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:453: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:433: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:491: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1304: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:937: There were test failures: 358 suites, 1577 tests, 2 suite-level errors, 229 ignored (6 assumptions)

Total time: 45 minutes 2 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0-ea-b121 -XX:-UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure