You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/11/22 21:11:10 UTC

[JENKINS] Lucene-Solr-trunk-Windows (32bit/jdk1.8.0-ea-b116) - Build # 3495 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Windows/3495/
Java: 32bit/jdk1.8.0-ea-b116 -client -XX:+UseParallelGC

5 tests failed.
REGRESSION:  org.apache.solr.cloud.DeleteInactiveReplicaTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:54939/pyew, http://127.0.0.1:54918/pyew]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:54939/pyew, http://127.0.0.1:54918/pyew]
	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC:F672F8E2641AFDE0]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:354)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:640)
	at org.apache.solr.cloud.DeleteReplicaTest.removeAndWaitForReplicaGone(DeleteReplicaTest.java:138)
	at org.apache.solr.cloud.DeleteInactiveReplicaTest.deleteInactiveReplicaTest(DeleteInactiveReplicaTest.java:120)
	at org.apache.solr.cloud.DeleteInactiveReplicaTest.doTest(DeleteInactiveReplicaTest.java:43)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: deletereplica the collection time out:60s
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:491)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:197)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:285)
	... 45 more


REGRESSION:  org.apache.solr.cloud.DeleteReplicaTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:55798, http://127.0.0.1:55786, http://127.0.0.1:55807, http://127.0.0.1:55816, http://127.0.0.1:55825]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:55798, http://127.0.0.1:55786, http://127.0.0.1:55807, http://127.0.0.1:55816, http://127.0.0.1:55825]
	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC:F672F8E2641AFDE0]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:354)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:640)
	at org.apache.solr.cloud.DeleteReplicaTest.removeAndWaitForReplicaGone(DeleteReplicaTest.java:138)
	at org.apache.solr.cloud.DeleteReplicaTest.deleteLiveReplicaTest(DeleteReplicaTest.java:126)
	at org.apache.solr.cloud.DeleteReplicaTest.doTest(DeleteReplicaTest.java:102)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: deletereplica the collection unkown case
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:491)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:197)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:285)
	... 45 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteReplicaTest

Error Message:
ERROR: SolrZkClient opens=15 closes=14

Stack Trace:
java.lang.AssertionError: ERROR: SolrZkClient opens=15 closes=14
	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingZkClients(SolrTestCaseJ4.java:343)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:135)
	at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:700)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteReplicaTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteReplicaTest:     1) Thread[id=3257, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-EventThread, state=WAITING, group=TGRP-DeleteReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)    2) Thread[id=3256, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-SendThread(127.0.0.1:55779), state=TIMED_WAITING, group=TGRP-DeleteReplicaTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteReplicaTest: 
   1) Thread[id=3257, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-EventThread, state=WAITING, group=TGRP-DeleteReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
   2) Thread[id=3256, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-SendThread(127.0.0.1:55779), state=TIMED_WAITING, group=TGRP-DeleteReplicaTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteReplicaTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=3256, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-SendThread(127.0.0.1:55779), state=TIMED_WAITING, group=TGRP-DeleteReplicaTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=3256, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-SendThread(127.0.0.1:55779), state=TIMED_WAITING, group=TGRP-DeleteReplicaTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC]:0)




Build Log:
[...truncated 9965 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteInactiveReplicaTest
   [junit4]   2> 327336 T1004 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /pyew/
   [junit4]   2> 327341 T1004 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-DeleteInactiveReplicaTest-1385149831558
   [junit4]   2> 327344 T1004 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 327345 T1005 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 327466 T1004 oasc.ZkTestServer.run start zk server on port:54911
   [junit4]   2> 327469 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 327477 T1011 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@127258d name:ZooKeeperConnection Watcher:127.0.0.1:54911 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 327477 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 327477 T1004 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 327484 T1006 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x142815d198f0000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 327488 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 327491 T1013 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b6c6fa name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 327492 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 327492 T1004 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 327498 T1004 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 327504 T1004 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 327508 T1004 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 327514 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 327515 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 327523 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 327523 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 327542 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 327542 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 327549 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 327549 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 327555 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 327555 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 327561 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 327561 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 327567 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 327567 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 327573 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 327573 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 327580 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 327580 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 327586 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 327586 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 327592 T1004 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 327592 T1004 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 328188 T1004 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 328202 T1004 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54918
   [junit4]   2> 328202 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 328202 T1004 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 328202 T1004 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819
   [junit4]   2> 328202 T1004 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\'
   [junit4]   2> 328240 T1004 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\solr.xml
   [junit4]   2> 328255 T1004 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 328257 T1004 oasc.CoreContainer.<init> New CoreContainer 26641681
   [junit4]   2> 328257 T1004 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\]
   [junit4]   2> 328258 T1004 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 328258 T1004 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 328258 T1004 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 328258 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 328259 T1004 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 328259 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 328259 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 328260 T1004 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 328260 T1004 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 328265 T1004 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 328266 T1004 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 328266 T1004 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 328266 T1004 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54911/solr
   [junit4]   2> 328267 T1004 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 328270 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 328276 T1024 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bd2eb2 name:ZooKeeperConnection Watcher:127.0.0.1:54911 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 328276 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 328283 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 328287 T1026 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b7edb4 name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 328287 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 328291 T1004 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 328298 T1004 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 328305 T1004 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 328310 T1004 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54918_pyew
   [junit4]   2> 328312 T1004 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54918_pyew
   [junit4]   2> 328322 T1004 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 328332 T1004 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 328338 T1004 oasc.Overseer.start Overseer (id=90777179361574915-127.0.0.1:54918_pyew-n_0000000000) starting
   [junit4]   2> 328346 T1004 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 328355 T1028 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 328356 T1004 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 328363 T1004 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 328367 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 328373 T1027 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 328375 T1004 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819
   [junit4]   2> 328680 T1004 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\conf\
   [junit4]   2> 328694 T1004 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 328696 T1004 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 328697 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 328704 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 328709 T1031 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1839208 name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 328710 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 328712 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 328720 T1004 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 329272 T1004 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 329279 T1004 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54930
   [junit4]   2> 329280 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 329280 T1004 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 329282 T1004 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935
   [junit4]   2> 329282 T1004 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\'
   [junit4]   2> 329317 T1004 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\solr.xml
   [junit4]   2> 329333 T1004 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 329334 T1004 oasc.CoreContainer.<init> New CoreContainer 10997412
   [junit4]   2> 329334 T1004 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\]
   [junit4]   2> 329335 T1004 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 329335 T1004 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 329335 T1004 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 329336 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 329336 T1004 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 329336 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 329337 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 329337 T1004 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 329337 T1004 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 329342 T1004 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 329343 T1004 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 329343 T1004 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 329343 T1004 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54911/solr
   [junit4]   2> 329345 T1004 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 329347 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329367 T1042 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@173632d name:ZooKeeperConnection Watcher:127.0.0.1:54911 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329368 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329377 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329380 T1044 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@111b7c8 name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329380 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329387 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 330608 T1004 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54930_pyew
   [junit4]   2> 330611 T1004 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54930_pyew
   [junit4]   2> 330619 T1026 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 330619 T1044 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 330620 T1031 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 330627 T1004 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935
   [junit4]   2> 330915 T1004 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\conf\
   [junit4]   2> 330930 T1004 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 330931 T1004 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 330931 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 331465 T1004 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 331470 T1004 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54939
   [junit4]   2> 331471 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 331472 T1004 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 331472 T1004 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385149835152
   [junit4]   2> 331473 T1004 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385149835152\'
   [junit4]   2> 331505 T1004 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385149835152\solr.xml
   [junit4]   2> 331519 T1004 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 331519 T1004 oasc.CoreContainer.<init> New CoreContainer 12906408
   [junit4]   2> 331520 T1004 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385149835152\]
   [junit4]   2> 331520 T1004 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 331521 T1004 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 331522 T1004 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 331522 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 331522 T1004 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 331523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 331523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 331523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 331523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 331529 T1004 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 331529 T1004 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 331529 T1004 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 331530 T1004 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54911/solr
   [junit4]   2> 331530 T1004 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 331533 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 331546 T1056 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66968e name:ZooKeeperConnection Watcher:127.0.0.1:54911 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 331546 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 331555 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 331558 T1058 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1177efc name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 331560 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 331567 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 332665 T1004 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54939_pyew
   [junit4]   2> 332667 T1004 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54939_pyew
   [junit4]   2> 332673 T1031 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 332674 T1058 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 332674 T1044 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 332674 T1026 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 332680 T1004 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385149835152
   [junit4]   2> 332955 T1004 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385149835152\conf\
   [junit4]   2> 332968 T1004 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 332970 T1004 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 332970 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 333454 T1004 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 333460 T1004 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54948
   [junit4]   2> 333461 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 333463 T1004 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 333463 T1004 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191
   [junit4]   2> 333463 T1004 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\'
   [junit4]   2> 333495 T1004 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\solr.xml
   [junit4]   2> 333508 T1004 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 333510 T1004 oasc.CoreContainer.<init> New CoreContainer 30863109
   [junit4]   2> 333510 T1004 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\]
   [junit4]   2> 333511 T1004 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 333511 T1004 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 333512 T1004 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 333512 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 333512 T1004 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 333512 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 333513 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 333513 T1004 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 333513 T1004 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 333517 T1004 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 333518 T1004 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 333518 T1004 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 333518 T1004 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54911/solr
   [junit4]   2> 333519 T1004 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 333521 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 333533 T1070 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16a2da7 name:ZooKeeperConnection Watcher:127.0.0.1:54911 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 333533 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 333545 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 333548 T1072 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bb7f35 name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 333548 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 333555 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 334654 T1004 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54948_pyew
   [junit4]   2> 334656 T1004 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54948_pyew
   [junit4]   2> 334664 T1031 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334665 T1058 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334665 T1044 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334666 T1026 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334667 T1072 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 334676 T1004 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191
   [junit4]   2> 334953 T1004 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\conf\
   [junit4]   2> 334965 T1004 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 334967 T1004 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 334967 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 335468 T1004 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 335473 T1004 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54957
   [junit4]   2> 335476 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 335476 T1004 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 335476 T1004 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186
   [junit4]   2> 335476 T1004 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186\'
   [junit4]   2> 335507 T1004 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186\solr.xml
   [junit4]   2> 335519 T1004 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 335519 T1004 oasc.CoreContainer.<init> New CoreContainer 18382488
   [junit4]   2> 335519 T1004 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186\]
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 335523 T1004 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 335528 T1004 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 335528 T1004 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 335528 T1004 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 335528 T1004 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54911/solr
   [junit4]   2> 335528 T1004 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 335532 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 335547 T1084 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ad5350 name:ZooKeeperConnection Watcher:127.0.0.1:54911 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 335548 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 335551 T1006 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x142815d198f000b, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 335555 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 335558 T1086 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@192e37b name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 335558 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 335564 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 336663 T1004 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54957_pyew
   [junit4]   2> 336665 T1004 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54957_pyew
   [junit4]   2> 336673 T1031 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 336674 T1086 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 336675 T1044 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 336675 T1072 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 336675 T1026 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 336675 T1058 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 336681 T1004 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186
   [junit4]   2> 336920 T1004 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186\conf\
   [junit4]   2> 336932 T1004 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 336934 T1004 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 336934 T1004 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 336944 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 336949 T1089 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@154c385 name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 336949 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 336951 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 336957 T1004 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 336960 T1091 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1af6d6f name:ZooKeeperConnection Watcher:127.0.0.1:54911/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 336961 T1004 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 336962 T1004 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 336967 T1049 oasha.CollectionsHandler.handleCreateAction Creating Collection : replicationFactor=2&maxShardsPerNode=2&name=delDeadColl&action=CREATE&numShards=2&wt=javabin&version=2
   [junit4]   2> 336971 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336973 T1028 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"delDeadColl",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 336973 T1028 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"delDeadColl",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 336977 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336979 T1027 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 336980 T1027 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: delDeadColl
   [junit4]   2> 336980 T1027 oasc.Overseer$ClusterStateUpdater.createCollection Create collection delDeadColl with shards [shard1, shard2]
   [junit4]   2> 336984 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 336987 T1091 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336988 T1072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336988 T1058 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336988 T1086 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336988 T1026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336989 T1044 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336990 T1089 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 336991 T1031 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337087 T1028 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 337087 T1028 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard1_replica1 as part of slice shard1 of collection delDeadColl on 127.0.0.1:54957_pyew
   [junit4]   2> 337089 T1028 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard1_replica2 as part of slice shard1 of collection delDeadColl on 127.0.0.1:54930_pyew
   [junit4]   2> 337091 T1028 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard2_replica1 as part of slice shard2 of collection delDeadColl on 127.0.0.1:54918_pyew
   [junit4]   2> 337092 T1028 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard2_replica2 as part of slice shard2 of collection delDeadColl on 127.0.0.1:54948_pyew
   [junit4]   2> 337103 T1080 oasc.ZkController.publish publishing core=delDeadColl_shard1_replica1 state=down
   [junit4]   2> 337103 T1040 oasc.ZkController.publish publishing core=delDeadColl_shard1_replica2 state=down
   [junit4]   2> 337105 T1065 oasc.ZkController.publish publishing core=delDeadColl_shard2_replica2 state=down
   [junit4]   2> 337106 T1018 oasc.ZkController.publish publishing core=delDeadColl_shard2_replica1 state=down
   [junit4]   2> 337107 T1080 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 337107 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 337108 T1040 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 337108 T1065 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 337109 T1018 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 337112 T1027 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 337113 T1027 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54957/pyew",
   [junit4]   2> 	  "core":"delDeadColl_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54957_pyew",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 337118 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 337119 T1027 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54930/pyew",
   [junit4]   2> 	  "core":"delDeadColl_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54930_pyew",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 337123 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 337125 T1027 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54948/pyew",
   [junit4]   2> 	  "core":"delDeadColl_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54948_pyew",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 337130 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 337131 T1027 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54918/pyew",
   [junit4]   2> 	  "core":"delDeadColl_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54918_pyew",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 337135 T1026 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 337248 T1091 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337249 T1086 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337249 T1089 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337250 T1031 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337250 T1058 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337250 T1026 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337251 T1072 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 337251 T1044 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 338201 T1080 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard1_replica1
   [junit4]   2> 338202 T1040 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard1_replica2
   [junit4]   2> 338203 T1080 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard1_replica1' using instanceDir: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186\delDeadColl_shard1_replica1
   [junit4]   2> 338203 T1080 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 338204 T1065 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard2_replica2
   [junit4]   2> 338203 T1040 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard1_replica2' using instanceDir: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2
   [junit4]   2> 338205 T1040 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 338206 T1080 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 338206 T1080 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 338204 T1018 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard2_replica1
   [junit4]   2> 338207 T1065 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard2_replica2' using instanceDir: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2
   [junit4]   2> 338207 T1065 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 338208 T1040 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 338208 T1040 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 338208 T1018 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard2_replica1' using instanceDir: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\delDeadColl_shard2_replica1
   [junit4]   2> 338209 T1018 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 338210 T1080 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 338211 T1080 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 338210 T1065 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 338211 T1065 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 338211 T1018 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 338213 T1018 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 338213 T1040 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 338214 T1040 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 338216 T1018 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 338217 T1018 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 338217 T1065 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 338217 T1065 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 338219 T1080 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 338220 T1040 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 338222 T1080 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385149839186\delDeadColl_shard1_replica1\'
   [junit4]   2> 338224 T1065 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 338226 T1018 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 338226 T1040 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2\'
   [junit4]   2> 338230 T1065 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2\'
   [junit4]   2> 338237 T1018 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\delDeadColl_shard2_replica1\'
   [junit4]   2> 338280 T1065 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 338303 T1065 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 338309 T1065 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 338313 T1018 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 338315 T1065 oass.IndexSchema.readSchema [delDeadColl_shard2_replica2] Schema name=test
   [junit4]   2> 338326 T1018 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 338338 T1080 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 338361 T1080 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 338371 T1040 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 338381 T1040 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 338389 T1018 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 338396 T1018 oass.IndexSchema.readSchema [delDeadColl_shard2_replica1] Schema name=test
   [junit4]   2> 338429 T1040 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 338437 T1040 oass.IndexSchema.readSchema [delDeadColl_shard1_replica2] Schema name=test
   [junit4]   2> 338475 T1065 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338483 T1065 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338484 T1065 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338576 T1080 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 338584 T1080 oass.IndexSchema.readSchema [delDeadColl_shard1_replica1] Schema name=test
   [junit4]   2> 338586 T1040 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338592 T1040 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338603 T1040 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338640 T1018 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338644 T1018 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338646 T1018 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338687 T1065 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338693 T1065 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338697 T1065 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338700 T1040 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338701 T1018 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338704 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338704 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338704 T1065 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338706 T1040 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338708 T1018 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 338711 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338711 T1065 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338711 T1065 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338712 T1065 oasc.SolrCore.<init> [delDeadColl_shard2_replica2] Opening new SolrCore at ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2\, dataDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2\data\
   [junit4]   2> 338712 T1065 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@94bc16
   [junit4]   2> 338713 T1065 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2\data\
   [junit4]   2> 338713 T1065 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2\data\index/
   [junit4]   2> 338713 T1065 oasc.SolrCore.initIndex WARN [delDeadColl_shard2_replica2] Solr index directory '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2\data\index' doesn't exist. Creating new index...
   [junit4]   2> 338714 T1065 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385149837191\delDeadColl_shard2_replica2\data\index
   [junit4]   2> 338717 T1065 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1174289364, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 338725 T1065 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1c2caaa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@10e005f),segFN=segments_1,generation=1}
   [junit4]   2> 338725 T1065 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 338730 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 338732 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 338732 T1065 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 338733 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 338733 T1065 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 338734 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 338734 T1065 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 338735 T1065 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 338735 T1065 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 338735 T1065 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 338736 T1065 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 338736 T1065 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 338736 T1065 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 338738 T1065 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 338738 T1065 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 338739 T1065 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 338744 T1065 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 338747 T1065 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 338749 T1065 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 338750 T1065 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=23, maxMergedSegmentMB=56.2861328125, floorSegmentMB=1.0498046875, forceMergeDeletesPctAllowed=19.827804372910467, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7752817282275389
   [junit4]   2> 338750 T1065 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1c2caaa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@10e005f),segFN=segments_1,generation=1}
   [junit4]   2> 338750 T1065 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 338750 T1065 oass.SolrIndexSearcher.<init> Opening Searcher@1d9395c main
   [junit4]   2> 338756 T1065 oasc.CoreContainer.registerCore registering core: delDeadColl_shard2_replica2
   [junit4]   2> 338757 T1065 oasc.ZkController.register Register replica - core:delDeadColl_shard2_replica2 address:http://127.0.0.1:54948/pyew collection:delDeadColl shard:shard2
   [junit4]   2> 338725 T1040 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338759 T1018 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338756 T1080 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 338760 T1065 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl/leader_elect/shard2/election
   [junit4]   2> 338764 T1040 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338765 T1040 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338765 T1040 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338766 T1080 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 338767 T1018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338767 T1018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338767 T1018 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 338768 T1080 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 338770 T1040 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338770 T1040 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338770 T1040 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338771 T1040 oasc.SolrCore.<init> [delDeadColl_shard1_replica2] Opening new SolrCore at ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2\, dataDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2\data\
   [junit4]   2> 338771 T1040 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@94bc16
   [junit4]   2> 338772 T1040 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2\data\
   [junit4]   2> 338773 T1040 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2\data\index/
   [junit4]   2> 338773 T1040 oasc.SolrCore.initIndex WARN [delDeadColl_shard1_replica2] Solr index directory '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2\data\index' doesn't exist. Creating new index...
   [junit4]   2> 338772 T1096 oasc.SolrCore.registerSearcher [delDeadColl_shard2_replica2] Registered new searcher Searcher@1d9395c main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 338774 T1040 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385149832935\delDeadColl_shard1_replica2\data\index
   [junit4]   2> 338774 T1018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 338775 T1018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 338776 T1018 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 338776 T1018 oasc.SolrCore.<init> [delDeadColl_shard2_replica1] Opening new SolrCore at ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\delDeadColl_shard2_replica1\, dataDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\delDeadColl_shard2_replica1\data\
   [junit4]   2> 338777 T1040 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1174289364, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 338777 T1018 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@94bc16
   [junit4]   2> 338780 T1018 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\delDeadColl_shard2_replica1\data\
   [junit4]   2> 338780 T1018 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\delDeadColl_shard2_replica1\data\index/
   [junit4]   2> 338780 T1018 oasc.SolrCore.initIndex WARN [delDeadColl_shard2_replica1] Solr index directory '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385149831819\delDeadColl_shard2_replica1\data\index' doesn't exist. Creating new index...
   [junit4]   2> 338781 T1018 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\or

[...truncated too long message...]

at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
   [junit4]   2> 
   [junit4]   2> 1482005 T3256 oaz.ClientCnxn$SendThread.run WARN Session 0x1428168f8dc000e for server null, unexpected error, closing socket connection and attempting reconnect java.nio.channels.ClosedByInterruptException
   [junit4]   2> 	at java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:744)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]   2> 
   [junit4]   2> 1482012 T3256 oaz.ClientCnxn$SendThread.run WARN Unexpected exception java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:984)
   [junit4]   2> 
   [junit4]   2> 377147 T3169 ccr.ThreadLeakControl.tryToInterruptAll SEVERE There are still zombie threads that couldn't be terminated:
   [junit4]   2> 	   1) Thread[id=3256, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-SendThread(127.0.0.1:55779), state=TIMED_WAITING, group=TGRP-DeleteReplicaTest]
   [junit4]   2> 	        at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
   [junit4]   2> NOTE: test params are: codec=Lucene46: {}, docValues:{}, sim=DefaultSimilarity, locale=, timezone=Africa/Banjul
   [junit4]   2> NOTE: Windows 7 6.1 x86/Oracle Corporation 1.8.0-ea (32-bit)/cpus=2,threads=3,free=75494576,total=263192576
   [junit4]   2> NOTE: All tests run in this JVM: [TestRealTimeGet, BadComponentTest, TestSimpleQParserPlugin, FunctionTest, TestValueSourceCache, URLClassifyProcessorTest, SimpleFacetsTest, SpellCheckCollatorTest, TestManagedSchema, TestAnalyzeInfixSuggestions, TestSolrXmlPersistence, TestWriterPerf, TestFieldTypeCollectionResource, TestLazyCores, TestStressUserVersions, DistributedDebugComponentTest, TestNoOpRegenerator, TestCSVLoader, TestFaceting, DistributedSpellCheckComponentTest, IndexBasedSpellCheckerTest, ZkSolrClientTest, TestLFUCache, TestSuggestSpellingConverter, MBeansHandlerTest, HdfsBasicDistributedZkTest, OpenExchangeRatesOrgProviderTest, UniqFieldsUpdateProcessorFactoryTest, ChaosMonkeySafeLeaderTest, DeleteInactiveReplicaTest, TestFastLRUCache, TestFastOutputStream, TestRecovery, CachingDirectoryFactoryTest, OverseerCollectionProcessorTest, TestLuceneMatchVersion, TestFiltering, NotRequiredUniqueKeyTest, DebugComponentTest, WordBreakSolrSpellCheckerTest, ScriptEngineTest, SolrTestCaseJ4Test, PingRequestHandlerTest, SolrCoreCheckLockOnStartupTest, TestBM25SimilarityFactory, PathHierarchyTokenizerFactoryTest, TestSolrDeletionPolicy2, TestPostingsSolrHighlighter, UnloadDistributedZkTest, PolyFieldTest, TestSearchPerf, ZkControllerTest, SimplePostToolTest, SolrInfoMBeanTest, SOLR749Test, TestStressLucene, TestRandomMergePolicy, ShardSplitTest, TestFunctionQuery, TestJmxMonitoredMap, TestDocSet, DocValuesMissingTest, OutputWriterTest, HardAutoCommitTest, CopyFieldTest, TestUpdate, ModifyConfFileTest, CurrencyFieldOpenExchangeTest, TestSolrXMLSerializer, DateMathParserTest, LeaderElectionIntegrationTest, SuggesterFSTTest, TestHashPartitioner, TestSolrDeletionPolicy1, QueryFacetTest, ReturnFieldsTest, TestDocumentBuilder, SolrXmlInZkTest, MoreLikeThisHandlerTest, TestIBSimilarityFactory, TestDistributedGrouping, TestQueryUtils, TestPHPSerializedResponseWriter, CoreContainerCoreInitFailuresTest, BinaryUpdateRequestHandlerTest, BJQParserTest, TestDynamicFieldCollectionResource, CollectionsAPIDistributedZkTest, ZkNodePropsTest, TestExtendedDismaxParser, DirectSolrSpellCheckerTest, TestPseudoReturnFields, FieldMutatingUpdateProcessorTest, StressHdfsTest, DistributedQueryElevationComponentTest, BadCopyFieldTest, TestSolrQueryParserResource, TestAtomicUpdateErrorCases, SpellingQueryConverterTest, TestRecoveryHdfs, TestUtils, TestSurroundQueryParser, TestSort, FieldFacetExtrasTest, DeleteShardTest, BlockCacheTest, TestMergePolicyConfig, TestCollapseQParserPlugin, TestSolrXmlPersistor, PrimitiveFieldTypeTest, ParsingFieldUpdateProcessorsTest, TestSolr4Spatial, HdfsBasicDistributedZk2Test, TermVectorComponentTest, DocumentAnalysisRequestHandlerTest, XmlUpdateRequestHandlerTest, FieldFacetTest, DisMaxRequestHandlerTest, TestMaxScoreQueryParser, ExpressionTest, MigrateRouteKeyTest, BasicFunctionalityTest, TestModifyConfFiles, TestStressVersions, TestGroupingSearch, DirectUpdateHandlerTest, EnumFieldTest, SolrIndexSplitterTest, TestRTGBase, TestCopyFieldCollectionResource, TestSolrXml, LukeRequestHandlerTest, CircularListTest, SignatureUpdateProcessorFactoryTest, MultiTermTest, SpellPossibilityIteratorTest, XsltUpdateRequestHandlerTest, CoreMergeIndexesAdminHandlerTest, TestClassNameShortening, TimeZoneUtilsTest, SliceStateTest, CoreAdminHandlerTest, TestQueryTypes, SolrIndexConfigTest, SuggesterTSTTest, SolrPluginUtilsTest, TestCloudManagedSchema, DeleteReplicaTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DeleteReplicaTest -Dtests.seed=779476FA13459DDC -Dtests.slow=true -Dtests.locale= -Dtests.timezone=Africa/Banjul -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s | DeleteReplicaTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ERROR: SolrZkClient opens=15 closes=14
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.endTrackingZkClients(SolrTestCaseJ4.java:343)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:135)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteReplicaTest: 
   [junit4]    >    1) Thread[id=3257, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-EventThread, state=WAITING, group=TGRP-DeleteReplicaTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
   [junit4]    >    2) Thread[id=3256, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-SendThread(127.0.0.1:55779), state=TIMED_WAITING, group=TGRP-DeleteReplicaTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=3256, name=TEST-DeleteReplicaTest.testDistribSearch-seed#[779476FA13459DDC]-SendThread(127.0.0.1:55779), state=TIMED_WAITING, group=TGRP-DeleteReplicaTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([779476FA13459DDC]:0)
   [junit4] Completed in 377.23s, 1 test, 1 failure, 3 errors <<< FAILURES!

[...truncated 596 lines...]
BUILD FAILED
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:420: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:400: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:39: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\extra-targets.xml:37: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build.xml:189: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\common-build.xml:489: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:1272: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:908: There were test failures: 349 suites, 1551 tests, 3 suite-level errors, 2 errors, 718 ignored (3 assumptions)

Total time: 49 minutes 53 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0-ea-b116 -client -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-trunk-Windows (32bit/jdk1.8.0-ea-b116) - Build # 3496 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Windows/3496/
Java: 32bit/jdk1.8.0-ea-b116 -server -XX:+UseConcMarkSweepGC

3 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=7622, name=HashSessionScavenger-235, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    2) Thread[id=7625, name=qtp12074201-7625, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:744)    3) Thread[id=7626, name=qtp12074201-7626, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:744)    4) Thread[id=7629, name=qtp12074201-7629, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:744)    5) Thread[id=7627, name=qtp12074201-7627, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:744)    6) Thread[id=7630, name=qtp12074201-7630, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:744)    7) Thread[id=7623, name=qtp12074201-7623 Acceptor0 SelectChannelConnector@127.0.0.1:59532, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)         at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:744)    8) Thread[id=7624, name=qtp12074201-7624, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:744)    9) Thread[id=7628, name=qtp12074201-7628 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:744)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 9 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=7622, name=HashSessionScavenger-235, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   2) Thread[id=7625, name=qtp12074201-7625, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:744)
   3) Thread[id=7626, name=qtp12074201-7626, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:744)
   4) Thread[id=7629, name=qtp12074201-7629, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:744)
   5) Thread[id=7627, name=qtp12074201-7627, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:744)
   6) Thread[id=7630, name=qtp12074201-7630, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:744)
   7) Thread[id=7623, name=qtp12074201-7623 Acceptor0 SelectChannelConnector@127.0.0.1:59532, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
        at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:744)
   8) Thread[id=7624, name=qtp12074201-7624, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:744)
   9) Thread[id=7628, name=qtp12074201-7628 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
        at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
        at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:744)
	at __randomizedtesting.SeedInfo.seed([4C1E9A235445D8F2]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=7622, name=HashSessionScavenger-235, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.util.TimerThread.mainLoop(Timer.java:552)         at java.util.TimerThread.run(Timer.java:505)    2) Thread[id=7623, name=qtp12074201-7623 Acceptor0 SelectChannelConnector@127.0.0.1:59532, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:744)    3) Thread[id=7628, name=qtp12074201-7628 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:105)         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536)         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:744)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=7622, name=HashSessionScavenger-235, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.util.TimerThread.mainLoop(Timer.java:552)
        at java.util.TimerThread.run(Timer.java:505)
   2) Thread[id=7623, name=qtp12074201-7623 Acceptor0 SelectChannelConnector@127.0.0.1:59532, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:744)
   3) Thread[id=7628, name=qtp12074201-7628 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
        at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:105)
        at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536)
        at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:744)
	at __randomizedtesting.SeedInfo.seed([4C1E9A235445D8F2]:0)


FAILED:  org.apache.solr.analytics.NoFacetTest.stddevTest

Error Message:
Oops: (double raws) 4631318898052956160 - 4628496337733101339 < 4442235333156365461 Calculated diff 4625071700926640586 Let's see what the JVM thinks these bits are. FloatResult:  43.5 floatTest: 27.72225134241226 Diff 15.77774865758774

Stack Trace:
java.lang.AssertionError: Oops: (double raws) 4631318898052956160 - 4628496337733101339 < 4442235333156365461 Calculated diff 4625071700926640586 Let's see what the JVM thinks these bits are. FloatResult:  43.5 floatTest: 27.72225134241226 Diff 15.77774865758774
	at __randomizedtesting.SeedInfo.seed([4C1E9A235445D8F2:49DEDA966426DCFD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.analytics.NoFacetTest.stddevTest(NoFacetTest.java:227)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)




Build Log:
[...truncated 10063 lines...]
   [junit4] Suite: org.apache.solr.analytics.NoFacetTest
   [junit4]   2> 246236 T1549 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> Creating dataDir: C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-NoFacetTest-1385167774804
   [junit4]   2> 246239 T1549 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: 'C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\'
   [junit4]   2> 246244 T1549 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/C:/Users/JenkinsSlave/workspace/Lucene-Solr-trunk-Windows/solr/build/solr-core/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 246247 T1549 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/C:/Users/JenkinsSlave/workspace/Lucene-Solr-trunk-Windows/solr/build/solr-core/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 246466 T1549 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 246471 T1549 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig-basic.xml
   [junit4]   2> 246474 T1549 oass.IndexSchema.readSchema Reading Solr Schema from schema-analytics.xml
   [junit4]   2> 246481 T1549 oass.IndexSchema.readSchema [null] Schema name=schema-docValues
   [junit4]   2> 246496 T1549 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 246497 T1549 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 246500 T1549 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr
   [junit4]   2> 246500 T1549 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: 'C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\'
   [junit4]   2> 246601 T1549 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 246601 T1549 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr
   [junit4]   2> 246601 T1549 oasc.SolrResourceLoader.<init> new SolrResourceLoader for deduced Solr Home: 'C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\'
   [junit4]   2> 246642 T1549 oasc.CoreContainer.<init> New CoreContainer 2274433
   [junit4]   2> 246642 T1549 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\]
   [junit4]   2> 246644 T1549 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 0
   [junit4]   2> 246644 T1549 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 246644 T1549 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 0
   [junit4]   2> 246644 T1549 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 246646 T1549 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 246646 T1549 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 246646 T1549 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 246646 T1549 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 246646 T1549 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 246655 T1549 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 246655 T1549 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 246656 T1549 oasc.CoreContainer.load Host Name: 
   [junit4]   2> 246670 T1550 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1
   [junit4]   2> 246670 T1550 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: 'C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\'
   [junit4]   2> 246670 T1550 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/C:/Users/JenkinsSlave/workspace/Lucene-Solr-trunk-Windows/solr/build/solr-core/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 246670 T1550 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/C:/Users/JenkinsSlave/workspace/Lucene-Solr-trunk-Windows/solr/build/solr-core/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 246727 T1550 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 246733 T1550 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig-basic.xml
   [junit4]   2> 246733 T1550 oass.IndexSchema.readSchema Reading Solr Schema from schema-analytics.xml
   [junit4]   2> 246742 T1550 oass.IndexSchema.readSchema [collection1] Schema name=schema-docValues
   [junit4]   2> 246771 T1550 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 246774 T1550 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 246775 T1550 oasc.SolrCore.<init> [collection1] Opening new SolrCore at C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\, dataDir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-NoFacetTest-1385167774804\
   [junit4]   2> 246775 T1550 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 246778 T1550 oasc.CachingDirectoryFactory.get return new directory for C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-NoFacetTest-1385167774804\
   [junit4]   2> 246778 T1550 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-NoFacetTest-1385167774804\index/
   [junit4]   2> 246778 T1550 oasc.SolrCore.initIndex WARN [collection1] Solr index directory 'C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-NoFacetTest-1385167774804\index' doesn't exist. Creating new index...
   [junit4]   2> 246794 T1550 oasc.CachingDirectoryFactory.get return new directory for C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-NoFacetTest-1385167774804\index
   [junit4]   2> 246794 T1550 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=82.23046875, floorSegmentMB=1.0654296875, forceMergeDeletesPctAllowed=3.029623864189511, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.22763952928783696
   [junit4]   2> 246794 T1550 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_1,generation=1}
   [junit4]   2> 246794 T1550 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 246797 T1550 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 246797 T1550 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 246798 T1550 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 246809 T1550 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 246809 T1550 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 246809 T1550 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 246810 T1550 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=72.5771484375, floorSegmentMB=0.6220703125, forceMergeDeletesPctAllowed=6.8864847320383955, segmentsPerTier=24.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 246810 T1550 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_1,generation=1}
   [junit4]   2> 246810 T1550 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 246810 T1550 oass.SolrIndexSearcher.<init> Opening Searcher@8897da main
   [junit4]   2> 246810 T1550 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 246817 T1549 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> ASYNC  NEW_CORE C804 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 246819 T1549 C804 oasu.DirectUpdateHandler2.deleteAll [collection1] REMOVING ALL DOCUMENTS FROM INDEX
   [junit4]   2> 246819 T1549 C804 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {deleteByQuery=*:*} 0 2
   [junit4]   2> 246823 T1549 C804 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10000]} 0 2
   [junit4]   2> 246828 T1549 C804 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 246831 T1549 C804 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_2,generation=2}
   [junit4]   2> 246839 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8897da main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 246839 T1549 C804 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 246840 T1549 C804 oass.SolrIndexSearcher.<init> Opening Searcher@3b376a main
   [junit4]   2> 246842 T1549 C804 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 246843 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3b376a main{StandardDirectoryReader(segments_2:4:nrt _0(5.0):c1)}
   [junit4]   2> 246843 T1549 C804 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 16
   [junit4]   2> 246850 T1549 C804 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10001]} 0 7
   [junit4]   2> 246851 T1549 C804 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 246931 T1549 C804 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_2,generation=2}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_3,generation=3}
   [junit4]   2> 246931 T1549 C804 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 3
   [junit4]   2> 246933 T1549 C804 oass.SolrIndexSearcher.<init> Opening Searcher@abf0a5 main
   [junit4]   2> 246933 T1549 C804 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 246935 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@abf0a5 main{StandardDirectoryReader(segments_3:6:nrt _0(5.0):c1 _1(5.0):c1)}
   [junit4]   2> 246935 T1549 C804 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 84
   [junit4]   2> 246935 T1549 C804 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10002]} 0 0
   [junit4]   2> 246939 T1549 C804 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 246940 T1549 C804 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_3,generation=3}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_4,generation=4}
   [junit4]   2> 246942 T1549 C804 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 4
   [junit4]   2> 246942 T1549 C804 oass.SolrIndexSearcher.<init> Opening Searcher@1bcd614 main
   [junit4]   2> 246942 T1549 C804 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 246942 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1bcd614 main{StandardDirectoryReader(segments_4:8:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1)}
   [junit4]   2> 246942 T1549 C804 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 4
   [junit4]   2> ASYNC  NEW_CORE C805 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 246954 T1549 C805 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10003]} 0 3
   [junit4]   2> 246955 T1549 C805 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 246959 T1549 C805 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_4,generation=4}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_5,generation=5}
   [junit4]   2> 246960 T1549 C805 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 5
   [junit4]   2> 246960 T1549 C805 oass.SolrIndexSearcher.<init> Opening Searcher@1589929 main
   [junit4]   2> 246960 T1549 C805 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 246966 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1589929 main{StandardDirectoryReader(segments_5:10:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1)}
   [junit4]   2> 246967 T1549 C805 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 12
   [junit4]   2> 246969 T1549 C805 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10004]} 0 2
   [junit4]   2> 246971 T1549 C805 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 246977 T1549 C805 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_5,generation=5}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_6,generation=6}
   [junit4]   2> 246977 T1549 C805 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 6
   [junit4]   2> 246978 T1549 C805 oass.SolrIndexSearcher.<init> Opening Searcher@1cd429c main
   [junit4]   2> 246981 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1cd429c main{StandardDirectoryReader(segments_6:12:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1)}
   [junit4]   2> 246981 T1549 C805 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 246981 T1549 C805 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 10
   [junit4]   2> 246983 T1549 C805 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10005]} 0 1
   [junit4]   2> 246988 T1549 C805 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 246993 T1549 C805 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_6,generation=6}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_7,generation=7}
   [junit4]   2> 246993 T1549 C805 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 7
   [junit4]   2> 246996 T1549 C805 oass.SolrIndexSearcher.<init> Opening Searcher@c8399b main
   [junit4]   2> 246998 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c8399b main{StandardDirectoryReader(segments_7:14:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1)}
   [junit4]   2> 246998 T1549 C805 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 246998 T1549 C805 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 12
   [junit4]   2> 247001 T1549 C805 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10006]} 0 1
   [junit4]   2> 247004 T1549 C805 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> ASYNC  NEW_CORE C806 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247016 T1549 C806 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_7,generation=7}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_8,generation=8}
   [junit4]   2> 247016 T1549 C806 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 8
   [junit4]   2> 247020 T1549 C806 oass.SolrIndexSearcher.<init> Opening Searcher@2f3da2 main
   [junit4]   2> 247021 T1549 C806 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247023 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2f3da2 main{StandardDirectoryReader(segments_8:16:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1)}
   [junit4]   2> 247023 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 21
   [junit4]   2> 247025 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10007]} 0 1
   [junit4]   2> 247028 T1549 C806 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247032 T1549 C806 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_8,generation=8}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_9,generation=9}
   [junit4]   2> 247033 T1549 C806 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 9
   [junit4]   2> 247036 T1549 C806 oass.SolrIndexSearcher.<init> Opening Searcher@2882ae main
   [junit4]   2> 247036 T1549 C806 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247038 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2882ae main{StandardDirectoryReader(segments_9:18:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1)}
   [junit4]   2> 247038 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 11
   [junit4]   2> 247040 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10008]} 0 1
   [junit4]   2> 247043 T1549 C806 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247047 T1549 C806 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_9,generation=9}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_a,generation=10}
   [junit4]   2> 247048 T1549 C806 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 10
   [junit4]   2> 247052 T1549 C806 oass.SolrIndexSearcher.<init> Opening Searcher@d41da2 main
   [junit4]   2> 247052 T1549 C806 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247054 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@d41da2 main{StandardDirectoryReader(segments_a:20:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1)}
   [junit4]   2> 247054 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 12
   [junit4]   2> 247056 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10009]} 0 2
   [junit4]   2> 247059 T1549 C806 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247063 T1549 C806 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_a,generation=10}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_b,generation=11}
   [junit4]   2> 247063 T1549 C806 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 11
   [junit4]   2> 247063 T1549 C806 oass.SolrIndexSearcher.<init> Opening Searcher@1087030 main
   [junit4]   2> 247067 T1549 C806 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247069 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1087030 main{StandardDirectoryReader(segments_b:22:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1)}
   [junit4]   2> 247069 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 11
   [junit4]   2> 247070 T1549 C806 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100010]} 0 1
   [junit4]   2> ASYNC  NEW_CORE C807 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247093 T1549 C807 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247100 T1549 C807 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_b,generation=11}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_c,generation=12}
   [junit4]   2> 247100 T1549 C807 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 12
   [junit4]   2> 247101 T1549 C807 oass.SolrIndexSearcher.<init> Opening Searcher@1c0b7d4 main
   [junit4]   2> 247101 T1549 C807 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247105 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1c0b7d4 main{StandardDirectoryReader(segments_c:24:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1)}
   [junit4]   2> 247105 T1549 C807 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 12
   [junit4]   2> 247106 T1549 C807 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100011]} 0 1
   [junit4]   2> 247111 T1549 C807 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247166 T1549 C807 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_c,generation=12}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_d,generation=13}
   [junit4]   2> 247166 T1549 C807 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 13
   [junit4]   2> 247189 T1549 C807 oass.SolrIndexSearcher.<init> Opening Searcher@29832d main
   [junit4]   2> 247189 T1549 C807 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247192 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29832d main{StandardDirectoryReader(segments_d:26:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1)}
   [junit4]   2> 247193 T1549 C807 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 84
   [junit4]   2> 247194 T1549 C807 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100012]} 0 1
   [junit4]   2> 247197 T1549 C807 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247202 T1549 C807 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_d,generation=13}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_e,generation=14}
   [junit4]   2> 247202 T1549 C807 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 14
   [junit4]   2> 247205 T1549 C807 oass.SolrIndexSearcher.<init> Opening Searcher@7f36de main
   [junit4]   2> 247205 T1549 C807 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247209 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7f36de main{StandardDirectoryReader(segments_e:28:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1)}
   [junit4]   2> 247209 T1549 C807 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 13
   [junit4]   2> 247211 T1549 C807 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100013]} 0 2
   [junit4]   2> 247213 T1549 C807 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247221 T1549 C807 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_e,generation=14}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_f,generation=15}
   [junit4]   2> 247221 T1549 C807 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 15
   [junit4]   2> 247223 T1549 C807 oass.SolrIndexSearcher.<init> Opening Searcher@11d2490 main
   [junit4]   2> 247227 T1549 C807 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247228 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@11d2490 main{StandardDirectoryReader(segments_f:30:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1)}
   [junit4]   2> 247228 T1549 C807 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 15
   [junit4]   2> ASYNC  NEW_CORE C808 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247239 T1549 C808 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100014]} 0 1
   [junit4]   2> 247242 T1549 C808 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247247 T1549 C808 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_f,generation=15}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_g,generation=16}
   [junit4]   2> 247248 T1549 C808 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 16
   [junit4]   2> 247251 T1549 C808 oass.SolrIndexSearcher.<init> Opening Searcher@1a40fb2 main
   [junit4]   2> 247252 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1a40fb2 main{StandardDirectoryReader(segments_g:32:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1)}
   [junit4]   2> 247254 T1549 C808 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247254 T1549 C808 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 14
   [junit4]   2> 247256 T1549 C808 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100015]} 0 2
   [junit4]   2> 247258 T1549 C808 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247262 T1549 C808 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_g,generation=16}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_h,generation=17}
   [junit4]   2> 247262 T1549 C808 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 17
   [junit4]   2> 247262 T1549 C808 oass.SolrIndexSearcher.<init> Opening Searcher@d0da3b main
   [junit4]   2> 247266 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@d0da3b main{StandardDirectoryReader(segments_h:34:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1)}
   [junit4]   2> 247266 T1549 C808 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247266 T1549 C808 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 8
   [junit4]   2> 247267 T1549 C808 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100016]} 0 1
   [junit4]   2> 247270 T1549 C808 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247277 T1549 C808 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_h,generation=17}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_i,generation=18}
   [junit4]   2> 247279 T1549 C808 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 18
   [junit4]   2> 247281 T1549 C808 oass.SolrIndexSearcher.<init> Opening Searcher@68d4fd main
   [junit4]   2> 247281 T1549 C808 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247285 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68d4fd main{StandardDirectoryReader(segments_i:36:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1)}
   [junit4]   2> 247286 T1549 C808 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 17
   [junit4]   2> 247292 T1549 C808 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100017]} 0 6
   [junit4]   2> 247294 T1549 C808 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> ASYNC  NEW_CORE C809 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247308 T1549 C809 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_i,generation=18}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_j,generation=19}
   [junit4]   2> 247308 T1549 C809 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 19
   [junit4]   2> 247311 T1549 C809 oass.SolrIndexSearcher.<init> Opening Searcher@3774a6 main
   [junit4]   2> 247311 T1549 C809 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247312 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3774a6 main{StandardDirectoryReader(segments_j:38:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1)}
   [junit4]   2> 247313 T1549 C809 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 20
   [junit4]   2> 247315 T1549 C809 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100018]} 0 2
   [junit4]   2> 247316 T1549 C809 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247320 T1549 C809 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_j,generation=19}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_k,generation=20}
   [junit4]   2> 247320 T1549 C809 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 20
   [junit4]   2> 247321 T1549 C809 oass.SolrIndexSearcher.<init> Opening Searcher@a5cbec main
   [junit4]   2> 247321 T1549 C809 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247323 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a5cbec main{StandardDirectoryReader(segments_k:40:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1)}
   [junit4]   2> 247324 T1549 C809 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 8
   [junit4]   2> 247327 T1549 C809 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100019]} 0 2
   [junit4]   2> 247328 T1549 C809 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247331 T1549 C809 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_k,generation=20}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_l,generation=21}
   [junit4]   2> 247331 T1549 C809 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 21
   [junit4]   2> 247335 T1549 C809 oass.SolrIndexSearcher.<init> Opening Searcher@3bbfc3 main
   [junit4]   2> 247335 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3bbfc3 main{StandardDirectoryReader(segments_l:42:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1)}
   [junit4]   2> 247335 T1549 C809 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247335 T1549 C809 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 8
   [junit4]   2> 247339 T1549 C809 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100020]} 0 3
   [junit4]   2> 247340 T1549 C809 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247361 T1549 C809 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_l,generation=21}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_m,generation=22}
   [junit4]   2> 247361 T1549 C809 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 22
   [junit4]   2> 247381 T1549 C809 oass.SolrIndexSearcher.<init> Opening Searcher@18030d0 main
   [junit4]   2> 247381 T1549 C809 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247382 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@18030d0 main{StandardDirectoryReader(segments_m:44:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1)}
   [junit4]   2> 247384 T1549 C809 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 45
   [junit4]   2> ASYNC  NEW_CORE C810 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247393 T1549 C810 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100021]} 0 1
   [junit4]   2> 247396 T1549 C810 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247401 T1549 C810 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_m,generation=22}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_n,generation=23}
   [junit4]   2> 247401 T1549 C810 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 23
   [junit4]   2> 247407 T1549 C810 oass.SolrIndexSearcher.<init> Opening Searcher@74d307 main
   [junit4]   2> 247407 T1549 C810 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247408 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@74d307 main{StandardDirectoryReader(segments_n:46:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1)}
   [junit4]   2> 247408 T1549 C810 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 14
   [junit4]   2> 247409 T1549 C810 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100022]} 0 1
   [junit4]   2> 247413 T1549 C810 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247423 T1549 C810 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_n,generation=23}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_o,generation=24}
   [junit4]   2> 247423 T1549 C810 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 24
   [junit4]   2> 247434 T1549 C810 oass.SolrIndexSearcher.<init> Opening Searcher@c2aee8 main
   [junit4]   2> 247434 T1549 C810 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247435 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c2aee8 main{StandardDirectoryReader(segments_o:48:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1)}
   [junit4]   2> 247435 T1549 C810 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 23
   [junit4]   2> 247436 T1549 C810 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100023]} 0 0
   [junit4]   2> 247439 T1549 C810 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247442 T1549 C810 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_o,generation=24}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_p,generation=25}
   [junit4]   2> 247442 T1549 C810 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 25
   [junit4]   2> 247444 T1549 C810 oass.SolrIndexSearcher.<init> Opening Searcher@b726e9 main
   [junit4]   2> 247444 T1549 C810 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247446 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@b726e9 main{StandardDirectoryReader(segments_p:50:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1)}
   [junit4]   2> 247447 T1549 C810 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 9
   [junit4]   2> 247448 T1549 C810 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100024]} 0 1
   [junit4]   2> 247451 T1549 C810 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> ASYNC  NEW_CORE C811 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247462 T1549 C811 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_p,generation=25}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_q,generation=26}
   [junit4]   2> 247462 T1549 C811 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 26
   [junit4]   2> 247462 T1549 C811 oass.SolrIndexSearcher.<init> Opening Searcher@d3f93e main
   [junit4]   2> 247462 T1549 C811 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247465 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@d3f93e main{StandardDirectoryReader(segments_q:52:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1)}
   [junit4]   2> 247465 T1549 C811 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 15
   [junit4]   2> 247466 T1549 C811 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100025]} 0 0
   [junit4]   2> 247469 T1549 C811 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247473 T1549 C811 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_q,generation=26}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_r,generation=27}
   [junit4]   2> 247473 T1549 C811 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 27
   [junit4]   2> 247473 T1549 C811 oass.SolrIndexSearcher.<init> Opening Searcher@196bf69 main
   [junit4]   2> 247473 T1549 C811 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247478 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@196bf69 main{StandardDirectoryReader(segments_r:54:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1)}
   [junit4]   2> 247478 T1549 C811 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 11
   [junit4]   2> 247480 T1549 C811 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100026]} 0 2
   [junit4]   2> 247481 T1549 C811 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247484 T1549 C811 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_r,generation=27}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_s,generation=28}
   [junit4]   2> 247486 T1549 C811 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 28
   [junit4]   2> 247486 T1549 C811 oass.SolrIndexSearcher.<init> Opening Searcher@15f1cc2 main
   [junit4]   2> 247489 T1549 C811 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247490 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@15f1cc2 main{StandardDirectoryReader(segments_s:56:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1)}
   [junit4]   2> 247490 T1549 C811 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 9
   [junit4]   2> 247493 T1549 C811 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100027]} 0 3
   [junit4]   2> 247494 T1549 C811 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247497 T1549 C811 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_s,generation=28}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_t,generation=29}
   [junit4]   2> 247500 T1549 C811 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 29
   [junit4]   2> 247500 T1549 C811 oass.SolrIndexSearcher.<init> Opening Searcher@1c011ad main
   [junit4]   2> 247504 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1c011ad main{StandardDirectoryReader(segments_t:58:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1)}
   [junit4]   2> 247504 T1549 C811 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247504 T1549 C811 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 10
   [junit4]   2> ASYNC  NEW_CORE C812 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247519 T1549 C812 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100028]} 0 2
   [junit4]   2> 247521 T1549 C812 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247528 T1549 C812 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_t,generation=29}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_u,generation=30}
   [junit4]   2> 247528 T1549 C812 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 30
   [junit4]   2> 247532 T1549 C812 oass.SolrIndexSearcher.<init> Opening Searcher@194e2b3 main
   [junit4]   2> 247532 T1549 C812 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247535 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@194e2b3 main{StandardDirectoryReader(segments_u:60:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1 _s(5.0):c1)}
   [junit4]   2> 247536 T1549 C812 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 16
   [junit4]   2> 247538 T1549 C812 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100029]} 0 2
   [junit4]   2> 247540 T1549 C812 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247547 T1549 C812 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_u,generation=30}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_v,generation=31}
   [junit4]   2> 247547 T1549 C812 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 31
   [junit4]   2> 247547 T1549 C812 oass.SolrIndexSearcher.<init> Opening Searcher@187d61f main
   [junit4]   2> 247551 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@187d61f main{StandardDirectoryReader(segments_v:62:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1 _s(5.0):c1 _t(5.0):c1)}
   [junit4]   2> 247551 T1549 C812 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247551 T1549 C812 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 11
   [junit4]   2> 247554 T1549 C812 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100030]} 0 3
   [junit4]   2> 247555 T1549 C812 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247559 T1549 C812 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_v,generation=31}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_w,generation=32}
   [junit4]   2> 247561 T1549 C812 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 32
   [junit4]   2> 247561 T1549 C812 oass.SolrIndexSearcher.<init> Opening Searcher@1dec1f main
   [junit4]   2> 247563 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1dec1f main{StandardDirectoryReader(segments_w:64:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1 _s(5.0):c1 _t(5.0):c1 _u(5.0):c1)}
   [junit4]   2> 247565 T1549 C812 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247565 T1549 C812 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 10
   [junit4]   2> 247566 T1549 C812 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100031]} 0 1
   [junit4]   2> ASYNC  NEW_CORE C813 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247574 T1549 C813 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247578 T1549 C813 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_w,generation=32}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_x,generation=33}
   [junit4]   2> 247578 T1549 C813 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 33
   [junit4]   2> 247578 T1549 C813 oass.SolrIndexSearcher.<init> Opening Searcher@1908c85 main
   [junit4]   2> 247584 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1908c85 main{StandardDirectoryReader(segments_x:66:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1 _s(5.0):c1 _t(5.0):c1 _u(5.0):c1 _v(5.0):c1)}
   [junit4]   2> 247584 T1549 C813 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247584 T1549 C813 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 10
   [junit4]   2> 247585 T1549 C813 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100032]} 0 0
   [junit4]   2> 247588 T1549 C813 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247590 T1549 C813 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_x,generation=33}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_y,generation=34}
   [junit4]   2> 247594 T1549 C813 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 34
   [junit4]   2> 247594 T1549 C813 oass.SolrIndexSearcher.<init> Opening Searcher@519a93 main
   [junit4]   2> 247596 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@519a93 main{StandardDirectoryReader(segments_y:68:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1 _s(5.0):c1 _t(5.0):c1 _u(5.0):c1 _v(5.0):c1 _w(5.0):c1)}
   [junit4]   2> 247597 T1549 C813 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247597 T1549 C813 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 11
   [junit4]   2> 247600 T1549 C813 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100033]} 0 3
   [junit4]   2> 247601 T1549 C813 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 247605 T1549 C813 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_y,generation=34}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_z,generation=35}
   [junit4]   2> 247605 T1549 C813 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 35
   [junit4]   2> 247605 T1549 C813 oass.SolrIndexSearcher.<init> Opening Searcher@9f7e90 main
   [junit4]   2> 247611 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9f7e90 main{StandardDirectoryReader(segments_z:70:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1 _s(5.0):c1 _t(5.0):c1 _u(5.0):c1 _v(5.0):c1 _w(5.0):c1 _x(5.0):c1)}
   [junit4]   2> 247611 T1549 C813 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 247611 T1549 C813 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 11
   [junit4]   2> 247612 T1549 C813 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[100034]} 0 0
   [junit4]   2> 247615 T1549 C813 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> ASYNC  NEW_CORE C814 name=collection1 org.apache.solr.core.SolrCore@46695
   [junit4]   2> 247623 T1549 C814 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_z,generation=35}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1ed3648 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1cdaa4f),segFN=segments_10,generation=36}
   [junit4]   2> 247627 T1549 C814 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 36
   [junit4]   2> 247628 T1549 C814 oass.SolrIndexSearcher.<init> Opening Searcher@cd1b6d main
   [junit4]   2> 247630 T1551 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@cd1b6d main{StandardDirectoryReader(segments_10:72:nrt _0(5.0):c1 _1(5.0):c1 _2(5.0):c1 _3(5.0):c1 _4(5.0):c1 _5(5.0):c1 _6(5.0):c1 _7(5.0):c1 _8(5.0):c1 _9(5.0):c1 _a(5.0):c1 _b(5.0):c1 _c(5.0):c1 _d(5.0):c1 _e(5.0):c1 _f(5.0):c1 _g(5.0):c1 _h(5.0):c1 _i(5.0):c1 _j(5.0):c1 _k(5.0):c1 _l(5.0):c1 _m(5.0):c1 _n(5.0):c1 _o(5.0):c1 _p(5.0):c1 _q(5.0):c1 _r(5.0):c1 _s(5.0):c1 _t(5.0):c1 _u(5

[...truncated too long message...]

ead[id=7629, name=qtp12074201-7629, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    >    5) Thread[id=7627, name=qtp12074201-7627, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    >    6) Thread[id=7630, name=qtp12074201-7630, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    >    7) Thread[id=7623, name=qtp12074201-7623 Acceptor0 SelectChannelConnector@127.0.0.1:59532, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
   [junit4]    >         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:241)
   [junit4]    >         at org.eclipse.jetty.server.nio.SelectChannelConnector.accept(SelectChannelConnector.java:109)
   [junit4]    >         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    >    8) Thread[id=7624, name=qtp12074201-7624, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
   [junit4]    >         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:342)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:526)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$600(QueuedThreadPool.java:44)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    >    9) Thread[id=7628, name=qtp12074201-7628 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.poll(WindowsSelectorImpl.java:296)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl$SubSelector.access$400(WindowsSelectorImpl.java:278)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:159)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:569)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4C1E9A235445D8F2]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=7622, name=HashSessionScavenger-235, state=TIMED_WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at java.lang.Object.wait(Native Method)
   [junit4]    >         at java.util.TimerThread.mainLoop(Timer.java:552)
   [junit4]    >         at java.util.TimerThread.run(Timer.java:505)
   [junit4]    >    2) Thread[id=7623, name=qtp12074201-7623 Acceptor0 SelectChannelConnector@127.0.0.1:59532, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:938)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    >    3) Thread[id=7628, name=qtp12074201-7628 Selector0, state=RUNNABLE, group=TGRP-ChaosMonkeyNothingIsSafeTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.setWakeupSocket(WindowsSelectorImpl.java:464)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.wakeup(WindowsSelectorImpl.java:606)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
   [junit4]    >         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:157)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
   [junit4]    >         at sun.nio.ch.SelectorImpl.selectNow(SelectorImpl.java:105)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$SelectSet.doSelect(SelectorManager.java:536)
   [junit4]    >         at org.eclipse.jetty.io.nio.SelectorManager$1.run(SelectorManager.java:290)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]    >         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]    >         at java.lang.Thread.run(Thread.java:744)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4C1E9A235445D8F2]:0)
   [junit4] Completed in 167.70s, 1 test, 2 errors <<< FAILURES!

[...truncated 184 lines...]
BUILD FAILED
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:420: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:400: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:39: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\extra-targets.xml:37: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build.xml:189: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\common-build.xml:489: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:1272: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:908: There were test failures: 349 suites, 1551 tests, 2 suite-level errors, 1 failure, 201 ignored (11 assumptions)

Total time: 64 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0-ea-b116 -server -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure