You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/10/29 19:56:52 UTC

[JENKINS] Lucene-Solr-trunk-Windows (32bit/jdk1.7.0_45) - Build # 3404 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Windows/3404/
Java: 32bit/jdk1.7.0_45 -client -XX:+UseConcMarkSweepGC

4 tests failed.
REGRESSION:  org.apache.solr.cloud.DeleteInactiveReplicaTest.testDistribSearch

Error Message:
Could not find jetty for replica core_node1:{"state":"recovering","base_url":"http://127.0.0.1:51624","core":"delDeadColl_shard1_replica1","node_name":"127.0.0.1:51624_"}jettys: http://127.0.0.1:51636/,http://127.0.0.1:51645/,http://127.0.0.1:51654/,http://127.0.0.1:51663/,

Stack Trace:
java.lang.AssertionError: Could not find jetty for replica core_node1:{"state":"recovering","base_url":"http://127.0.0.1:51624","core":"delDeadColl_shard1_replica1","node_name":"127.0.0.1:51624_"}jettys: http://127.0.0.1:51636/,http://127.0.0.1:51645/,http://127.0.0.1:51654/,http://127.0.0.1:51663/,
	at __randomizedtesting.SeedInfo.seed([5A8F70CA7A6902FC:DB69FED20D3662C0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.DeleteInactiveReplicaTest.deleteInactiveReplicaTest(DeleteInactiveReplicaTest.java:63)
	at org.apache.solr.cloud.DeleteInactiveReplicaTest.doTest(DeleteInactiveReplicaTest.java:38)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteInactiveReplicaTest

Error Message:
ERROR: SolrZkClient opens=15 closes=14

Stack Trace:
java.lang.AssertionError: ERROR: SolrZkClient opens=15 closes=14
	at __randomizedtesting.SeedInfo.seed([5A8F70CA7A6902FC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingZkClients(SolrTestCaseJ4.java:343)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:135)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:700)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteInactiveReplicaTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteInactiveReplicaTest:     1) Thread[id=1149, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=TIMED_WAITING, group=TGRP-DeleteInactiveReplicaTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:984)    2) Thread[id=1150, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-EventThread, state=WAITING, group=TGRP-DeleteInactiveReplicaTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteInactiveReplicaTest: 
   1) Thread[id=1149, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=TIMED_WAITING, group=TGRP-DeleteInactiveReplicaTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:984)
   2) Thread[id=1150, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-EventThread, state=WAITING, group=TGRP-DeleteInactiveReplicaTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
	at __randomizedtesting.SeedInfo.seed([5A8F70CA7A6902FC]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.DeleteInactiveReplicaTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=1149, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=RUNNABLE, group=TGRP-DeleteInactiveReplicaTest]         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=1149, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=RUNNABLE, group=TGRP-DeleteInactiveReplicaTest]
        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
	at __randomizedtesting.SeedInfo.seed([5A8F70CA7A6902FC]:0)




Build Log:
[...truncated 9784 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteInactiveReplicaTest
   [junit4]   2> 360254 T1063 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 360254 T1063 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\solrtest-DeleteInactiveReplicaTest-1383072874788
   [junit4]   2> 360269 T1063 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 360269 T1064 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 360379 T1063 oasc.ZkTestServer.run start zk server on port:51617
   [junit4]   2> 360379 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 360394 T1070 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f37b4 name:ZooKeeperConnection Watcher:127.0.0.1:51617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 360394 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 360394 T1063 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 360410 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 360410 T1065 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x142059141430000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 360410 T1072 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ac4ce3 name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 360410 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 360410 T1063 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 360410 T1063 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 360426 T1063 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 360426 T1063 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 360426 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 360426 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 360441 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 360441 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 360457 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 360457 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 360457 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 360457 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 360472 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 360472 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 360472 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 360472 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 360488 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 360488 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 360504 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 360504 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 360504 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 360504 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 360519 T1063 oasc.AbstractZkTestCase.putConfig put C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test-files\solr\collection1\conf\synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 360519 T1063 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 360535 T1065 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x142059141430001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 361035 T1063 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 361051 T1063 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51624
   [junit4]   2> 361051 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 361051 T1063 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 361051 T1063 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069
   [junit4]   2> 361051 T1063 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\'
   [junit4]   2> 361097 T1063 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\solr.xml
   [junit4]   2> 361222 T1063 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 361222 T1063 oasc.CoreContainer.<init> New CoreContainer 30310536
   [junit4]   2> 361222 T1063 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\]
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 361222 T1063 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 361222 T1063 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 361222 T1063 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 361222 T1063 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51617/solr
   [junit4]   2> 361222 T1063 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 361238 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 361238 T1083 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14c26f9 name:ZooKeeperConnection Watcher:127.0.0.1:51617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 361238 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 361238 T1065 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x142059141430002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 361238 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 361254 T1085 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1aafcf7 name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 361254 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 361254 T1063 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 361254 T1063 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 361269 T1063 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 361269 T1063 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51624_
   [junit4]   2> 361269 T1063 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51624_
   [junit4]   2> 361269 T1063 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 361285 T1063 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 361285 T1063 oasc.Overseer.start Overseer (id=90641063924137987-127.0.0.1:51624_-n_0000000000) starting
   [junit4]   2> 361301 T1063 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 361301 T1087 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 361316 T1063 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 361316 T1063 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 361316 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 361332 T1086 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 361347 T1063 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069
   [junit4]   2> 361566 T1063 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\conf\
   [junit4]   2> 361582 T1063 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 361582 T1063 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 361582 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 361597 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 361597 T1090 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77857 name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 361597 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 361597 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 361613 T1063 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 362113 T1063 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 362129 T1063 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51636
   [junit4]   2> 362129 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 362129 T1063 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 362129 T1063 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1383072876147
   [junit4]   2> 362129 T1063 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1383072876147\'
   [junit4]   2> 362176 T1063 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1383072876147\solr.xml
   [junit4]   2> 362285 T1063 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 362285 T1063 oasc.CoreContainer.<init> New CoreContainer 6058254
   [junit4]   2> 362285 T1063 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1383072876147\]
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 362301 T1063 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 362301 T1063 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 362301 T1063 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 362301 T1063 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51617/solr
   [junit4]   2> 362301 T1063 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 362301 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 362316 T1101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ccf48c name:ZooKeeperConnection Watcher:127.0.0.1:51617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 362316 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 362363 T1065 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x142059141430005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 362363 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 362379 T1103 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7c6cb7 name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 362379 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 362394 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 363394 T1063 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51636_
   [junit4]   2> 363394 T1063 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51636_
   [junit4]   2> 363410 T1090 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 363410 T1085 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 363410 T1103 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 363426 T1063 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1383072876147
   [junit4]   2> 363691 T1063 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1383072876147\conf\
   [junit4]   2> 363707 T1063 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 363707 T1063 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 363707 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 364222 T1063 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 364222 T1063 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51645
   [junit4]   2> 364222 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 364222 T1063 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 364222 T1063 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241
   [junit4]   2> 364222 T1063 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\'
   [junit4]   2> 364269 T1063 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\solr.xml
   [junit4]   2> 364363 T1063 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 364363 T1063 oasc.CoreContainer.<init> New CoreContainer 32162557
   [junit4]   2> 364363 T1063 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\]
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 364379 T1063 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 364379 T1063 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 364379 T1063 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 364379 T1063 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51617/solr
   [junit4]   2> 364379 T1063 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 364379 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 364394 T1115 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d1f59f name:ZooKeeperConnection Watcher:127.0.0.1:51617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 364394 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 364410 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 364410 T1117 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ac98e8 name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 364410 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 364426 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 365426 T1063 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51645_
   [junit4]   2> 365426 T1063 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51645_
   [junit4]   2> 365441 T1090 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 365441 T1103 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 365441 T1085 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 365441 T1117 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 365472 T1063 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241
   [junit4]   2> 365691 T1063 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\conf\
   [junit4]   2> 365707 T1063 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 365707 T1063 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 365707 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 366269 T1063 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 366269 T1063 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51654
   [junit4]   2> 366269 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 366269 T1063 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 366269 T1063 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256
   [junit4]   2> 366269 T1063 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\'
   [junit4]   2> 366316 T1063 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\solr.xml
   [junit4]   2> 366426 T1063 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 366426 T1063 oasc.CoreContainer.<init> New CoreContainer 24755311
   [junit4]   2> 366426 T1063 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\]
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 366441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 366441 T1063 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 366441 T1063 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 366441 T1063 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51617/solr
   [junit4]   2> 366441 T1063 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 366441 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 366457 T1129 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1641d7d name:ZooKeeperConnection Watcher:127.0.0.1:51617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 366457 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 366472 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 366472 T1131 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17f7e1b name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 366472 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 366488 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 367488 T1063 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51654_
   [junit4]   2> 367488 T1063 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51654_
   [junit4]   2> 367504 T1090 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 367504 T1085 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 367504 T1103 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 367504 T1131 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 367504 T1117 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 367519 T1063 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256
   [junit4]   2> 367769 T1063 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\conf\
   [junit4]   2> 367785 T1063 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 367785 T1063 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 367785 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 368254 T1063 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 368269 T1063 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51663
   [junit4]   2> 368269 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 368269 T1063 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 368269 T1063 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319
   [junit4]   2> 368269 T1063 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\'
   [junit4]   2> 368316 T1063 oasc.ConfigSolr.fromFile Loading container configuration from C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\solr.xml
   [junit4]   2> 368426 T1063 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 368426 T1063 oasc.CoreContainer.<init> New CoreContainer 1458530
   [junit4]   2> 368426 T1063 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\]
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 368441 T1063 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 368441 T1063 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 368441 T1063 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 368441 T1063 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51617/solr
   [junit4]   2> 368441 T1063 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 368441 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 368441 T1143 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@964717 name:ZooKeeperConnection Watcher:127.0.0.1:51617 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 368457 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 368457 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 368457 T1145 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f7571f name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 368457 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 368472 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 369472 T1063 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51663_
   [junit4]   2> 369472 T1063 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51663_
   [junit4]   2> 369488 T1145 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 369488 T1131 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 369488 T1085 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 369488 T1117 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 369488 T1103 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 369488 T1090 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 369519 T1063 oasc.CorePropertiesLocator.discover Looking for core definitions underneath C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319
   [junit4]   2> 369722 T1063 oasc.CorePropertiesLocator.discoverUnder Found core conf in C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\conf\
   [junit4]   2> 369754 T1063 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 369754 T1063 oass.SolrDispatchFilter.init user.dir=C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0
   [junit4]   2> 369754 T1063 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 369754 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 369769 T1148 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1691c14 name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 369769 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 369769 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 369785 T1063 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 369785 T1150 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@9feb64 name:ZooKeeperConnection Watcher:127.0.0.1:51617/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 369785 T1063 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 369785 T1063 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 369801 T1109 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&maxShardsPerNode=2&name=delDeadColl&replicationFactor=2&action=CREATE&wt=javabin&version=2
   [junit4]   2> 369801 T1085 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 369816 T1087 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"delDeadColl",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 369816 T1087 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"delDeadColl",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 370332 T1086 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 370332 T1086 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: delDeadColl
   [junit4]   2> 370332 T1086 oasc.Overseer$ClusterStateUpdater.createCollection Create collection delDeadColl with shards [shard1, shard2]
   [junit4]   2> 370363 T1117 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370363 T1090 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370363 T1150 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370363 T1085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370363 T1131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370363 T1145 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370363 T1103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370363 T1148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 370472 T1087 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 370472 T1087 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard1_replica1 as part of slice shard1 of collection delDeadColl on 127.0.0.1:51624_
   [junit4]   2> 370472 T1087 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard1_replica2 as part of slice shard1 of collection delDeadColl on 127.0.0.1:51654_
   [junit4]   2> 370472 T1087 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard2_replica1 as part of slice shard2 of collection delDeadColl on 127.0.0.1:51663_
   [junit4]   2> 370472 T1087 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard2_replica2 as part of slice shard2 of collection delDeadColl on 127.0.0.1:51645_
   [junit4]   2> 370488 T1081 oasc.ZkController.publish publishing core=delDeadColl_shard1_replica1 state=down
   [junit4]   2> 370488 T1126 oasc.ZkController.publish publishing core=delDeadColl_shard1_replica2 state=down
   [junit4]   2> 370488 T1111 oasc.ZkController.publish publishing core=delDeadColl_shard2_replica2 state=down
   [junit4]   2> 370488 T1141 oasc.ZkController.publish publishing core=delDeadColl_shard2_replica1 state=down
   [junit4]   2> 370488 T1081 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 370488 T1126 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 370488 T1141 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 370504 T1111 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 371879 T1086 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 371879 T1086 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51624",
   [junit4]   2> 	  "core":"delDeadColl_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51624_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 371894 T1086 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51654",
   [junit4]   2> 	  "core":"delDeadColl_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51654_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 371894 T1086 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51663",
   [junit4]   2> 	  "core":"delDeadColl_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51663_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 371910 T1086 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51645",
   [junit4]   2> 	  "core":"delDeadColl_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51645_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 371910 T1117 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 371910 T1103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 371910 T1145 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 371910 T1148 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 371926 T1090 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 371926 T1150 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 371926 T1085 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 371910 T1131 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 372488 T1126 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard1_replica2
   [junit4]   2> 372488 T1081 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard1_replica1
   [junit4]   2> 372488 T1126 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard1_replica2' using instanceDir: ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2
   [junit4]   2> 372488 T1126 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 372488 T1081 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard1_replica1' using instanceDir: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1
   [junit4]   2> 372488 T1081 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 372488 T1126 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 372488 T1081 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 372488 T1081 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 372488 T1126 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 372488 T1081 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 372488 T1126 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 372488 T1126 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 372488 T1081 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 372504 T1141 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard2_replica1
   [junit4]   2> 372504 T1111 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard2_replica2
   [junit4]   2> 372504 T1081 oasc.ZkController.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 372504 T1141 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard2_replica1' using instanceDir: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1
   [junit4]   2> 372504 T1141 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 372504 T1126 oasc.ZkController.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 372504 T1111 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard2_replica2' using instanceDir: .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\delDeadColl_shard2_replica2
   [junit4]   2> 372504 T1111 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 372504 T1141 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 372504 T1141 oasc.ZkController.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 372504 T1111 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 372504 T1111 oasc.ZkController.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 372504 T1081 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1\'
   [junit4]   2> 372519 T1141 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1\'
   [junit4]   2> 372519 T1126 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2\'
   [junit4]   2> 372519 T1111 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\delDeadColl_shard2_replica2\'
   [junit4]   2> 372691 T1081 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 372769 T1126 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 372847 T1111 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 372863 T1141 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 372941 T1126 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 372972 T1126 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 372988 T1126 oass.IndexSchema.readSchema [delDeadColl_shard1_replica2] Schema name=test
   [junit4]   2> 372988 T1081 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 373066 T1141 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 373113 T1081 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 373129 T1141 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 373144 T1081 oass.IndexSchema.readSchema [delDeadColl_shard1_replica1] Schema name=test
   [junit4]   2> 373144 T1141 oass.IndexSchema.readSchema [delDeadColl_shard2_replica1] Schema name=test
   [junit4]   2> 373238 T1111 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 373332 T1111 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 373363 T1111 oass.IndexSchema.readSchema [delDeadColl_shard2_replica2] Schema name=test
   [junit4]   2> 375004 T1126 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 375051 T1126 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 375051 T1126 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 375488 T1141 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 375566 T1141 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 375582 T1141 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 375613 T1081 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 375629 T1081 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 375629 T1081 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 375644 T1126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 375660 T1126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 375676 T1126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 375676 T1126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 375707 T1126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 375707 T1126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 375816 T1126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 375816 T1126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 375816 T1126 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 375816 T1126 oasc.SolrCore.<init> [delDeadColl_shard1_replica2] Opening new SolrCore at ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2\, dataDir=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2\data\
   [junit4]   2> 375816 T1126 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f2f78f
   [junit4]   2> 375816 T1126 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2\data\
   [junit4]   2> 375816 T1126 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2\data\index/
   [junit4]   2> 375816 T1126 oasc.SolrCore.initIndex WARN [delDeadColl_shard1_replica2] Solr index directory '..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2\data\index' doesn't exist. Creating new index...
   [junit4]   2> 375816 T1126 oasc.CachingDirectoryFactory.get return new directory for ..\..\..\..\..\..\..\..\..\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build\solr-core\test\J0\.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1383072880256\delDeadColl_shard1_replica2\data\index
   [junit4]   2> 375816 T1126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.11042678988050617]
   [junit4]   2> 375816 T1126 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@198f2a1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2779b),segFN=segments_1,generation=1}
   [junit4]   2> 375816 T1126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 375816 T1126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 375832 T1126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 375832 T1126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 375832 T1126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 375832 T1126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 375832 T1126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 375832 T1126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 375832 T1126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 375832 T1126 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 375832 T1126 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 375832 T1126 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 375832 T1126 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 375832 T1126 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 375832 T1126 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 375832 T1126 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 375847 T1126 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 375863 T1126 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 375863 T1126 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 375863 T1126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1810053942, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 375863 T1126 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@198f2a1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2779b),segFN=segments_1,generation=1}
   [junit4]   2> 375863 T1126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 375863 T1126 oass.SolrIndexSearcher.<init> Opening Searcher@19197b6 main
   [junit4]   2> 375863 T1141 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 375879 T1141 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 375894 T1126 oasc.CoreContainer.registerCore registering core: delDeadColl_shard1_replica2
   [junit4]   2> 375894 T1126 oasc.ZkController.register Register replica - core:delDeadColl_shard1_replica2 address:http://127.0.0.1:51654 collection:delDeadColl shard:shard1
   [junit4]   2> 375894 T1155 oasc.SolrCore.registerSearcher [delDeadColl_shard1_replica2] Registered new searcher Searcher@19197b6 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 375894 T1081 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 375894 T1126 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl/leader_elect/shard1/election
   [junit4]   2> 375910 T1141 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 375926 T1081 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 375926 T1141 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 375926 T1141 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 375941 T1141 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 375941 T1081 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 375941 T1141 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 375941 T1141 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 375941 T1141 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 375941 T1141 oasc.SolrCore.<init> [delDeadColl_shard2_replica1] Opening new SolrCore at .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1\, dataDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1\data\
   [junit4]   2> 375941 T1141 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f2f78f
   [junit4]   2> 375957 T1141 oasc.CachingDirectoryFactory.get return new directory for .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1\data\
   [junit4]   2> 375957 T1141 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1\data\index/
   [junit4]   2> 375957 T1141 oasc.SolrCore.initIndex WARN [delDeadColl_shard2_replica1] Solr index directory '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1\data\index' doesn't exist. Creating new index...
   [junit4]   2> 375957 T1141 oasc.CachingDirectoryFactory.get return new directory for .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1383072882319\delDeadColl_shard2_replica1\data\index
   [junit4]   2> 375957 T1141 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.11042678988050617]
   [junit4]   2> 375957 T1141 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@45f1c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4502a9),segFN=segments_1,generation=1}
   [junit4]   2> 375957 T1141 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 375957 T1141 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 375972 T1141 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 375972 T1141 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 375972 T1141 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 375972 T1141 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 375972 T1141 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 375972 T1141 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 375972 T1141 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 375972 T1141 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 375972 T1141 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 375972 T1141 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 375972 T1141 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 375972 T1141 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 375972 T1141 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 375972 T1141 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 375972 T1141 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 375988 T1081 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 375988 T1081 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 375988 T1081 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 375988 T1141 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 375988 T1141 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 375988 T1141 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1810053942, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 375988 T1141 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@45f1c8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4502a9),segFN=segments_1,generation=1}
   [junit4]   2> 375988 T1141 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 375988 T1141 oass.SolrIndexSearcher.<init> Opening Searcher@32aa9c main
   [junit4]   2> 376004 T1141 oasc.CoreContainer.registerCore registering core: delDeadColl_shard2_replica1
   [junit4]   2> 376004 T1141 oasc.ZkController.register Register replica - core:delDeadColl_shard2_replica1 address:http://127.0.0.1:51663 collection:delDeadColl shard:shard2
   [junit4]   2> 376004 T1156 oasc.SolrCore.registerSearcher [delDeadColl_shard2_replica1] Registered new searcher Searcher@32aa9c main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 376004 T1081 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 376004 T1081 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 376004 T1081 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 376004 T1081 oasc.SolrCore.<init> [delDeadColl_shard1_replica1] Opening new SolrCore at .\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1\, dataDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1\data\
   [junit4]   2> 376004 T1081 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f2f78f
   [junit4]   2> 376019 T1081 oasc.CachingDirectoryFactory.get return new directory for .\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1\data\
   [junit4]   2> 376019 T1081 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1\data\index/
   [junit4]   2> 376019 T1081 oasc.SolrCore.initIndex WARN [delDeadColl_shard1_replica1] Solr index directory '.\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1\data\index' doesn't exist. Creating new index...
   [junit4]   2> 376019 T1081 oasc.CachingDirectoryFactory.get return new directory for .\org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1383072875069\delDeadColl_shard1_replica1\data\index
   [junit4]   2> 376019 T1081 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.11042678988050617]
   [junit4]   2> 376019 T1081 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1a9917d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1637cf7),segFN=segments_1,generation=1}
   [junit4]   2> 376019 T1081 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 376019 T1081 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 376019 T1081 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 376019 T1141 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl/leader_elect/shard2/election
   [junit4]   2> 376019 T1081 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 376019 T1081 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 376019 T1081 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 376019 T1081 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 376035 T1081 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 376035 T1081 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 376035 T1081 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 376035 T1081 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 376035 T1081 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 376035 T1081 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 376035 T1111 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 376035 T1081 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 376035 T1081 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 376035 T1081 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 376035 T1081 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 376051 T1111 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 376051 T1081 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 376051 T1081 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 376051 T1081 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1810053942, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 376051 T1081 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1a9917d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1637cf7),segFN=segments_1,generation=1}
   [junit4]   2> 376051 T1081 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 376051 T1081 oass.SolrIndexSearcher.<init> Opening Searcher@f4a5c7 main
   [junit4]   2> 376051 T1111 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 376051 T1126 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 376066 T1157 oasc.SolrCore.registerSearcher [delDeadColl_shard1_replica1] Registered new searcher Searcher@f4a5c7 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 376066 T1081 oasc.CoreContainer.registerCore registering core: delDeadColl_shard1_replica1
   [junit4]   2> 376066 T1081 oasc.ZkController.register Register replica - core:delDeadColl_shard1_replica1 address:http://127.0.0.1:51624 collection:delDeadColl shard:shard1
   [junit4]   2> 376066 T1126 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=180000
   [junit4]   2> 376082 T1141 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 376082 T1141 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=180000
   [junit4]   2> 376097 T1111 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 376097 T1111 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 376113 T1111 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 376113 T1111 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 376113 T1111 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 376113 T1111 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 376113 T1111 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 376113 T1111 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 376113 T1111 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 376113 T1111 oasc.SolrCore.<init> [delDeadColl_shard2_replica2] Opening new SolrCore at .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\delDeadColl_shard2_replica2\, dataDir=.\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\delDeadColl_shard2_replica2\data\
   [junit4]   2> 376113 T1111 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@f2f78f
   [junit4]   2> 376113 T1111 oasc.CachingDirectoryFactory.get return new directory for .\org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1383072878241\delDeadColl_shard2_replica2\data\
   [jun

[...truncated too long message...]

-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=TIMED_WAITING, group=TGRP-DeleteInactiveReplicaTest]
   [junit4]   2> 	   2) Thread[id=1150, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-EventThread, state=WAITING, group=TGRP-DeleteInactiveReplicaTest]
   [junit4]   2> 414359 T1150 oaz.ClientCnxn$EventThread.run ERROR Event thread exiting due to interruption java.lang.InterruptedException
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2017)
   [junit4]   2> 	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2052)
   [junit4]   2> 	at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
   [junit4]   2> 
   [junit4]   2> 414360 T1149 oaz.ClientCnxn$SendThread.run WARN Unexpected exception java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:984)
   [junit4]   2> 
   [junit4]   2> 414860 T1149 oazc.StaticHostProvider.next WARN Unexpected exception java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
   [junit4]   2> 
   [junit4]   2> 415872 T1149 oaz.ClientCnxn$SendThread.run WARN Session 0x14205914143000e for server null, unexpected error, closing socket connection and attempting reconnect java.nio.channels.ClosedByInterruptException
   [junit4]   2> 	at java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:771)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]   2> 
   [junit4]   2> 416360 T1149 oazc.StaticHostProvider.next WARN Unexpected exception java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 	at java.lang.Thread.sleep(Native Method)
   [junit4]   2> 	at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:86)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:937)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:993)
   [junit4]   2> 
   [junit4]   2> 57107 T1062 ccr.ThreadLeakControl.tryToInterruptAll SEVERE There are still zombie threads that couldn't be terminated:
   [junit4]   2> 	   1) Thread[id=1149, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=RUNNABLE, group=TGRP-DeleteInactiveReplicaTest]
   [junit4]   2> 	        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
   [junit4]   2> 	        at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
   [junit4]   2> 	        at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=ar, timezone=Pacific/Wake
   [junit4]   2> NOTE: Windows 7 6.1 x86/Oracle Corporation 1.7.0_45 (32-bit)/cpus=2,threads=2,free=112512904,total=172408832
   [junit4]   2> NOTE: All tests run in this JVM: [TestCollationField, SpellCheckComponentTest, TestQuerySenderNoQuery, SampleTest, SolrCoreCheckLockOnStartupTest, BJQParserTest, TestNoOpRegenerator, TestStressRecovery, TestSolrCoreProperties, BlockDirectoryTest, SolrPluginUtilsTest, DistributedSpellCheckComponentTest, TestCopyFieldCollectionResource, TestReplicationHandler, LeaderElectionIntegrationTest, JSONWriterTest, TestSchemaSimilarityResource, TestStressVersions, SolrTestCaseJ4Test, CoreMergeIndexesAdminHandlerTest, TestNonNRTOpen, BinaryUpdateRequestHandlerTest, SearchHandlerTest, TestSolrDeletionPolicy1, DeleteInactiveReplicaTest]
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DeleteInactiveReplicaTest -Dtests.seed=5A8F70CA7A6902FC -Dtests.slow=true -Dtests.locale=ar -Dtests.timezone=Pacific/Wake -Dtests.file.encoding=US-ASCII
   [junit4]   2> 417399 T1149 oaz.ClientCnxn$SendThread.run WARN Session 0x14205914143000e for server null, unexpected error, closing socket connection and attempting reconnect java.nio.channels.ClosedByInterruptException
   [junit4]   2> 	at java.nio.channels.spi.AbstractInterruptibleChannel.end(AbstractInterruptibleChannel.java:202)
   [junit4]   2> 	at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:771)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:350)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]   2> 
   [junit4] ERROR   0.00s | DeleteInactiveReplicaTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ERROR: SolrZkClient opens=15 closes=14
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5A8F70CA7A6902FC]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.endTrackingZkClients(SolrTestCaseJ4.java:343)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:135)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.DeleteInactiveReplicaTest: 
   [junit4]    >    1) Thread[id=1149, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=TIMED_WAITING, group=TGRP-DeleteInactiveReplicaTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:984)
   [junit4]    >    2) Thread[id=1150, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-EventThread, state=WAITING, group=TGRP-DeleteInactiveReplicaTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:491)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5A8F70CA7A6902FC]:0)Throwable #3: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=1149, name=TEST-DeleteInactiveReplicaTest.testDistribSearch-seed#[5A8F70CA7A6902FC]-SendThread(127.0.0.1:51617), state=RUNNABLE, group=TGRP-DeleteInactiveReplicaTest]
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket0(Native Method)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.resetWakeupSocket(WindowsSelectorImpl.java:473)
   [junit4]    >         at sun.nio.ch.WindowsSelectorImpl.doSelect(WindowsSelectorImpl.java:174)
   [junit4]    >         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   [junit4]    >         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   [junit4]    >         at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:338)
   [junit4]    >         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1068)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5A8F70CA7A6902FC]:0)
   [junit4] Completed in 57.21s, 1 test, 2 failures, 2 errors <<< FAILURES!

[...truncated 916 lines...]
BUILD FAILED
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:422: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:402: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\build.xml:39: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\build.xml:189: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\solr\common-build.xml:483: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:1259: The following error occurred while executing this line:
C:\Users\JenkinsSlave\workspace\Lucene-Solr-trunk-Windows\lucene\common-build.xml:902: There were test failures: 329 suites, 1453 tests, 3 suite-level errors, 1 failure, 1288 ignored (2 assumptions)

Total time: 35 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.7.0_45 -client -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure