You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/11/23 12:29:37 UTC

[JENKINS] Lucene-Solr-4.x-Linux (32bit/jdk1.7.0_45) - Build # 8333 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/8333/
Java: 32bit/jdk1.7.0_45 -client -XX:+UseParallelGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.DeleteInactiveReplicaTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:51419/jm/af, http://127.0.0.1:44753/jm/af]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:51419/jm/af, http://127.0.0.1:44753/jm/af]
	at __randomizedtesting.SeedInfo.seed([D5FB414AFE20FA10:541DCF52897F9A2C]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:354)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:640)
	at org.apache.solr.cloud.DeleteReplicaTest.removeAndWaitForReplicaGone(DeleteReplicaTest.java:138)
	at org.apache.solr.cloud.DeleteInactiveReplicaTest.deleteInactiveReplicaTest(DeleteInactiveReplicaTest.java:120)
	at org.apache.solr.cloud.DeleteInactiveReplicaTest.doTest(DeleteInactiveReplicaTest.java:43)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: deletereplica the collection time out:60s
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:491)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:197)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:285)
	... 45 more




Build Log:
[...truncated 10971 lines...]
   [junit4] Suite: org.apache.solr.cloud.DeleteInactiveReplicaTest
   [junit4]   2> 1323258 T4719 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /jm/af
   [junit4]   2> 1323262 T4719 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-DeleteInactiveReplicaTest-1385205806739
   [junit4]   2> 1323263 T4719 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1323264 T4720 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1323364 T4719 oasc.ZkTestServer.run start zk server on port:51362
   [junit4]   2> 1323365 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1323428 T4726 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a15199 name:ZooKeeperConnection Watcher:127.0.0.1:51362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1323428 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1323429 T4719 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1323434 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1323435 T4728 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1052571 name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1323435 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1323435 T4719 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1323438 T4719 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1323440 T4719 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1323442 T4719 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1323444 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1323445 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1323451 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1323452 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1323455 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1323456 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1323458 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1323459 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1323461 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1323461 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1323464 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1323465 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1323467 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1323467 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1323470 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1323470 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1323473 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1323473 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1323477 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1323478 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1323482 T4719 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1323483 T4719 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1323562 T4719 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1323566 T4719 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51419
   [junit4]   2> 1323566 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1323567 T4719 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1323567 T4719 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967
   [junit4]   2> 1323567 T4719 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/'
   [junit4]   2> 1323579 T4719 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/solr.xml
   [junit4]   2> 1323617 T4719 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1323618 T4719 oasc.CoreContainer.<init> New CoreContainer 16469511
   [junit4]   2> 1323618 T4719 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/]
   [junit4]   2> 1323619 T4719 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1323619 T4719 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1323620 T4719 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1323620 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1323620 T4719 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1323621 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1323621 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1323621 T4719 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1323622 T4719 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1323623 T4719 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1323623 T4719 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1323623 T4719 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1323624 T4719 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51362/solr
   [junit4]   2> 1323624 T4719 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1323625 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1323627 T4739 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@147b0ac name:ZooKeeperConnection Watcher:127.0.0.1:51362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1323627 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1323629 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1323631 T4741 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1929f7e name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1323631 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1323632 T4719 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1323635 T4719 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1323637 T4719 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1323638 T4719 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51419_jm%2Faf
   [junit4]   2> 1323639 T4719 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51419_jm%2Faf
   [junit4]   2> 1323641 T4719 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1323645 T4719 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1323647 T4719 oasc.Overseer.start Overseer (id=90780847750578179-127.0.0.1:51419_jm%2Faf-n_0000000000) starting
   [junit4]   2> 1323649 T4719 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1323652 T4743 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1323653 T4719 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1323654 T4719 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1323656 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1323658 T4742 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1323660 T4719 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967
   [junit4]   2> 1323661 T4719 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/conf/
   [junit4]   2> 1323703 T4719 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1323704 T4719 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1323704 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1323706 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1323708 T4746 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1dd5189 name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1323708 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1323709 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1323710 T4719 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1323779 T4719 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1323780 T4719 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:44753
   [junit4]   2> 1323782 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1323782 T4719 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1323782 T4719 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385205807187
   [junit4]   2> 1323783 T4719 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385205807187/'
   [junit4]   2> 1323794 T4719 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385205807187/solr.xml
   [junit4]   2> 1323832 T4719 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1323833 T4719 oasc.CoreContainer.<init> New CoreContainer 7581506
   [junit4]   2> 1323833 T4719 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385205807187/]
   [junit4]   2> 1323834 T4719 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1323835 T4719 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1323835 T4719 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1323835 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1323836 T4719 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1323836 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1323836 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1323837 T4719 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1323837 T4719 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1323838 T4719 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1323838 T4719 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1323839 T4719 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1323839 T4719 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51362/solr
   [junit4]   2> 1323839 T4719 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1323840 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1323841 T4757 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15c3354 name:ZooKeeperConnection Watcher:127.0.0.1:51362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1323842 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1323844 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1323845 T4759 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10cf6f8 name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1323845 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1323848 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1324851 T4719 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44753_jm%2Faf
   [junit4]   2> 1324854 T4719 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44753_jm%2Faf
   [junit4]   2> 1324858 T4746 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1324859 T4759 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1324859 T4741 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1324866 T4719 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385205807187
   [junit4]   2> 1324868 T4719 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty1-1385205807187/conf/
   [junit4]   2> 1324901 T4719 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1324901 T4719 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1324902 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1324970 T4719 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1324972 T4719 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:56204
   [junit4]   2> 1324973 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1324974 T4719 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1324974 T4719 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381
   [junit4]   2> 1324974 T4719 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/'
   [junit4]   2> 1324986 T4719 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/solr.xml
   [junit4]   2> 1325025 T4719 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1325025 T4719 oasc.CoreContainer.<init> New CoreContainer 31965417
   [junit4]   2> 1325026 T4719 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/]
   [junit4]   2> 1325027 T4719 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1325028 T4719 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1325028 T4719 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1325029 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1325029 T4719 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1325029 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1325030 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1325030 T4719 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1325030 T4719 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1325032 T4719 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1325032 T4719 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1325032 T4719 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1325033 T4719 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51362/solr
   [junit4]   2> 1325033 T4719 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1325034 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1325035 T4771 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b511a name:ZooKeeperConnection Watcher:127.0.0.1:51362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1325036 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1325038 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1325040 T4773 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@126babb name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1325040 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1325045 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1326048 T4719 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56204_jm%2Faf
   [junit4]   2> 1326050 T4719 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56204_jm%2Faf
   [junit4]   2> 1326055 T4746 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1326055 T4773 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1326055 T4741 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1326055 T4759 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1326065 T4719 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381
   [junit4]   2> 1326066 T4719 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/conf/
   [junit4]   2> 1326098 T4719 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1326099 T4719 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1326099 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1326164 T4719 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1326173 T4719 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:44029
   [junit4]   2> 1326174 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1326175 T4719 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1326175 T4719 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576
   [junit4]   2> 1326176 T4719 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/'
   [junit4]   2> 1326188 T4719 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/solr.xml
   [junit4]   2> 1326225 T4719 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1326226 T4719 oasc.CoreContainer.<init> New CoreContainer 22234359
   [junit4]   2> 1326226 T4719 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/]
   [junit4]   2> 1326227 T4719 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1326227 T4719 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1326228 T4719 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1326228 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1326228 T4719 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1326228 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1326229 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1326229 T4719 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1326229 T4719 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1326231 T4719 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1326231 T4719 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1326231 T4719 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1326232 T4719 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51362/solr
   [junit4]   2> 1326232 T4719 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1326233 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1326235 T4785 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@782c11 name:ZooKeeperConnection Watcher:127.0.0.1:51362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1326235 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1326238 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1326239 T4787 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ae8958 name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1326239 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1326242 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1327245 T4719 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44029_jm%2Faf
   [junit4]   2> 1327246 T4719 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44029_jm%2Faf
   [junit4]   2> 1327248 T4773 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1327248 T4741 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1327248 T4759 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1327248 T4746 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1327249 T4787 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1327255 T4719 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576
   [junit4]   2> 1327257 T4719 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/conf/
   [junit4]   2> 1327303 T4719 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1327304 T4719 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1327304 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1327370 T4719 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1327373 T4719 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:37058
   [junit4]   2> 1327373 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1327374 T4719 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1327374 T4719 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782
   [junit4]   2> 1327374 T4719 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782/'
   [junit4]   2> 1327386 T4719 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782/solr.xml
   [junit4]   2> 1327424 T4719 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1327425 T4719 oasc.CoreContainer.<init> New CoreContainer 10146997
   [junit4]   2> 1327425 T4719 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782/]
   [junit4]   2> 1327427 T4719 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1327427 T4719 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1327427 T4719 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1327428 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1327428 T4719 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1327428 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1327429 T4719 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1327429 T4719 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1327429 T4719 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1327431 T4719 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1327431 T4719 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1327431 T4719 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1327432 T4719 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51362/solr
   [junit4]   2> 1327432 T4719 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1327433 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1327434 T4799 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1617af5 name:ZooKeeperConnection Watcher:127.0.0.1:51362 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1327434 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1327436 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1327438 T4801 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@642791 name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1327438 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1327442 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1328445 T4719 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37058_jm%2Faf
   [junit4]   2> 1328447 T4719 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37058_jm%2Faf
   [junit4]   2> 1328452 T4773 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1328453 T4746 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1328453 T4787 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1328453 T4759 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1328452 T4741 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1328453 T4801 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1328464 T4719 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782
   [junit4]   2> 1328466 T4719 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782/conf/
   [junit4]   2> 1328507 T4719 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1328508 T4719 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1328508 T4719 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1328510 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1328511 T4804 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c7b59 name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1328511 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1328512 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1328515 T4719 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1328516 T4806 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c310c7 name:ZooKeeperConnection Watcher:127.0.0.1:51362/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1328517 T4719 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1328518 T4719 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1328521 T4735 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&maxShardsPerNode=2&name=delDeadColl&replicationFactor=2&action=CREATE&wt=javabin&version=2
   [junit4]   2> 1328522 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328523 T4743 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"delDeadColl",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1328523 T4743 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"delDeadColl",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1328529 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328531 T4742 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1328533 T4742 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: delDeadColl
   [junit4]   2> 1328533 T4742 oasc.Overseer$ClusterStateUpdater.createCollection Create collection delDeadColl with shards [shard1, shard2]
   [junit4]   2> 1328538 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328539 T4773 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328539 T4741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328539 T4806 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328539 T4759 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328539 T4804 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328539 T4801 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328539 T4787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328539 T4746 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328629 T4743 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 1328630 T4743 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard1_replica1 as part of slice shard1 of collection delDeadColl on 127.0.0.1:51419_jm%2Faf
   [junit4]   2> 1328630 T4743 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard1_replica2 as part of slice shard1 of collection delDeadColl on 127.0.0.1:44029_jm%2Faf
   [junit4]   2> 1328632 T4743 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard2_replica1 as part of slice shard2 of collection delDeadColl on 127.0.0.1:37058_jm%2Faf
   [junit4]   2> 1328633 T4743 oasc.OverseerCollectionProcessor.createCollection Creating shard delDeadColl_shard2_replica2 as part of slice shard2 of collection delDeadColl on 127.0.0.1:56204_jm%2Faf
   [junit4]   2> 1328635 T4737 oasc.ZkController.publish publishing core=delDeadColl_shard1_replica1 state=down
   [junit4]   2> 1328635 T4780 oasc.ZkController.publish publishing core=delDeadColl_shard1_replica2 state=down
   [junit4]   2> 1328636 T4795 oasc.ZkController.publish publishing core=delDeadColl_shard2_replica1 state=down
   [junit4]   2> 1328636 T4769 oasc.ZkController.publish publishing core=delDeadColl_shard2_replica2 state=down
   [junit4]   2> 1328638 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328638 T4737 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1328639 T4780 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1328640 T4769 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1328640 T4795 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1328642 T4742 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1328644 T4742 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51419/jm/af",
   [junit4]   2> 	  "core":"delDeadColl_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:51419_jm%2Faf",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1328648 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328650 T4742 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44029/jm/af",
   [junit4]   2> 	  "core":"delDeadColl_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:44029_jm%2Faf",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1328656 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328658 T4742 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37058/jm/af",
   [junit4]   2> 	  "core":"delDeadColl_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37058_jm%2Faf",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1328663 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328665 T4742 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56204/jm/af",
   [junit4]   2> 	  "core":"delDeadColl_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56204_jm%2Faf",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"delDeadColl",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1328668 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1328772 T4806 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328772 T4746 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328772 T4773 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328772 T4804 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328772 T4759 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328772 T4787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328772 T4801 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1328772 T4741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1329639 T4737 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard1_replica1
   [junit4]   2> 1329640 T4780 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard1_replica2
   [junit4]   2> 1329641 T4737 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard1_replica1' using instanceDir: ./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1
   [junit4]   2> 1329642 T4780 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard1_replica2' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2
   [junit4]   2> 1329642 T4780 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 1329641 T4795 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard2_replica1
   [junit4]   2> 1329641 T4769 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for delDeadColl_shard2_replica2
   [junit4]   2> 1329644 T4780 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 1329645 T4769 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard2_replica2' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2
   [junit4]   2> 1329646 T4769 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 1329642 T4737 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 1329646 T4780 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1329645 T4795 oasc.CoreContainer.create Creating SolrCore 'delDeadColl_shard2_replica1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782/delDeadColl_shard2_replica1
   [junit4]   2> 1329648 T4769 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 1329651 T4769 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1329650 T4795 oasc.ZkController.createCollectionZkNode Check for collection zkNode:delDeadColl
   [junit4]   2> 1329651 T4737 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 1329653 T4795 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:delDeadColl
   [junit4]   2> 1329651 T4780 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1329654 T4769 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1329654 T4795 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1329653 T4737 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1329655 T4769 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 1329655 T4780 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 1329658 T4795 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1329659 T4795 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 1329660 T4737 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1329661 T4737 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl
   [junit4]   2> 1329663 T4795 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 1329663 T4769 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 1329664 T4780 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 1329665 T4795 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty4-1385205810782/delDeadColl_shard2_replica1/'
   [junit4]   2> 1329669 T4769 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2/'
   [junit4]   2> 1329671 T4737 oascc.ZkStateReader.readConfigName Load collection config from:/collections/delDeadColl
   [junit4]   2> 1329672 T4780 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2/'
   [junit4]   2> 1329675 T4737 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1/'
   [junit4]   2> 1329752 T4795 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1329754 T4737 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1329779 T4780 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1329780 T4769 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1329837 T4795 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1329840 T4737 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1329844 T4737 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1329845 T4795 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1329861 T4737 oass.IndexSchema.readSchema [delDeadColl_shard1_replica1] Schema name=test
   [junit4]   2> 1329862 T4769 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1329863 T4780 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1329864 T4769 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1329866 T4780 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1329866 T4795 oass.IndexSchema.readSchema [delDeadColl_shard2_replica1] Schema name=test
   [junit4]   2> 1329881 T4780 oass.IndexSchema.readSchema [delDeadColl_shard1_replica2] Schema name=test
   [junit4]   2> 1329882 T4769 oass.IndexSchema.readSchema [delDeadColl_shard2_replica2] Schema name=test
   [junit4]   2> 1330641 T4780 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330642 T4780 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330643 T4780 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1330655 T4780 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1330660 T4780 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1330675 T4780 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330679 T4780 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330683 T4780 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330683 T4769 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330685 T4769 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330685 T4780 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1330686 T4780 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1330686 T4769 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1330686 T4780 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330688 T4780 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1330688 T4780 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1330689 T4780 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1330689 T4780 oasc.SolrCore.<init> [delDeadColl_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2/data/
   [junit4]   2> 1330690 T4780 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6d48c1
   [junit4]   2> 1330691 T4780 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2/data
   [junit4]   2> 1330692 T4780 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2/data/index/
   [junit4]   2> 1330692 T4780 oasc.SolrCore.initIndex WARN [delDeadColl_shard1_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1330693 T4780 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty3-1385205809576/delDeadColl_shard1_replica2/data/index
   [junit4]   2> 1330693 T4780 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=87.3369140625, floorSegmentMB=0.4033203125, forceMergeDeletesPctAllowed=14.362544361475301, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1330694 T4780 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@5435db lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@8599ad),segFN=segments_1,generation=1}
   [junit4]   2> 1330695 T4780 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1330695 T4769 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1330699 T4780 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1330700 T4780 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1330701 T4780 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1330701 T4780 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1330702 T4780 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1330702 T4780 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1330703 T4780 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1330704 T4780 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1330704 T4780 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1330700 T4769 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1330715 T4737 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330717 T4737 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330717 T4737 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1330720 T4780 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1330722 T4780 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1330722 T4780 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1330722 T4780 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1330723 T4769 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330723 T4780 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1330724 T4737 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1330724 T4780 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1330726 T4780 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1330727 T4737 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1330728 T4769 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330730 T4780 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1330734 T4769 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330735 T4780 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1330736 T4780 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1330736 T4769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1330737 T4769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1330737 T4769 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330737 T4780 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=13.875, floorSegmentMB=1.19140625, forceMergeDeletesPctAllowed=28.19049544242447, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.36574365566423817
   [junit4]   2> 1330739 T4780 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@5435db lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@8599ad),segFN=segments_1,generation=1}
   [junit4]   2> 1330739 T4780 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1330739 T4769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1330740 T4769 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1330739 T4780 oass.SolrIndexSearcher.<init> Opening Searcher@c6aee7 main
   [junit4]   2> 1330740 T4769 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1330741 T4769 oasc.SolrCore.<init> [delDeadColl_shard2_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2/data/
   [junit4]   2> 1330742 T4769 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6d48c1
   [junit4]   2> 1330743 T4737 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330743 T4769 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2/data
   [junit4]   2> 1330743 T4769 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2/data/index/
   [junit4]   2> 1330744 T4769 oasc.SolrCore.initIndex WARN [delDeadColl_shard2_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1330745 T4769 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.DeleteInactiveReplicaTest-jetty2-1385205808381/delDeadColl_shard2_replica2/data/index
   [junit4]   2> 1330745 T4811 oasc.SolrCore.registerSearcher [delDeadColl_shard1_replica2] Registered new searcher Searcher@c6aee7 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1330745 T4769 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=87.3369140625, floorSegmentMB=0.4033203125, forceMergeDeletesPctAllowed=14.362544361475301, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1330747 T4769 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@823700 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@95b83a),segFN=segments_1,generation=1}
   [junit4]   2> 1330747 T4769 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1330748 T4737 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330750 T4780 oasc.CoreContainer.registerCore registering core: delDeadColl_shard1_replica2
   [junit4]   2> 1330751 T4780 oasc.ZkController.register Register replica - core:delDeadColl_shard1_replica2 address:http://127.0.0.1:44029/jm/af collection:delDeadColl shard:shard1
   [junit4]   2> 1330752 T4780 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl/leader_elect/shard1/election
   [junit4]   2> 1330752 T4769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1330754 T4737 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330757 T4769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1330757 T4737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1330759 T4737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1330759 T4769 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1330761 T4780 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1330760 T4737 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330761 T4769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1330764 T4780 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=179999
   [junit4]   2> 1330763 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1330765 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1330766 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1330763 T4737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1330767 T4737 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1330767 T4737 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1330768 T4737 oasc.SolrCore.<init> [delDeadColl_shard1_replica1] Opening new SolrCore at ./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1/, dataDir=./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1/data/
   [junit4]   2> 1330768 T4737 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6d48c1
   [junit4]   2> 1330764 T4769 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1330767 T4742 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1330770 T4737 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1/data
   [junit4]   2> 1330770 T4769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1330772 T4769 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1330772 T4769 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1330773 T4769 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1330774 T4769 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1330774 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1330775 T4737 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1/data/index/
   [junit4]   2> 1330775 T4737 oasc.SolrCore.initIndex WARN [delDeadColl_shard1_replica1] Solr index directory './org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1330775 T4769 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1330777 T4746 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4801 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4804 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4773 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4759 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4787 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4806 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4741 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1330777 T4737 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.DeleteInactiveReplicaTest-controljetty-1385205806967/delDeadColl_shard1_replica1/data/index
   [junit4]   2> 1330780 T4737 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=87.3369140625, floorSegmentMB=0.4033203125, forceMergeDeletesPctAllowed=14.362544361475301, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1330777 T4769 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1330781 T4769 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1330782 T4769 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1330783 T4769 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1330784 T4769 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1330786 T4737 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@11fb92b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@18e4db5),segFN=segments_1,generation=1}
   [junit4]   2> 1330786 T4737 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1330786 T4769 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1330800 T4737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1330801 T4737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1330802 T4737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1330803 T4737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1330803 T4737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1330803 T4737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1330804 T4737 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1330805 T4737 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1330805 T4737 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1330806 T4737 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1330808 T4737 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1330808 T4737 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1330809 T4737 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1330810 T4737 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1330810 T4737 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1330811 T4769 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1330811 T4737 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1330811 T4769 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1330813 T4769 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=13.875, floorSegmentMB=1.19140625, forceMergeDeletesPctAllowed=28.19049544242447, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.36574365566423817
   [junit4]   2> 1330814 T4737 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1330814 T4769 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@823700 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@95b83a),segFN=segments_1,generation=1}
   [junit4]   2> 1330815 T4769 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1330815 T4769 oass.SolrIndexSearcher.<init> Opening Searcher@178c038 main
   [junit4]   2> 1330819 T4737 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1330819 T4812 oasc.SolrCore.registerSearcher [delDeadColl_shard2_replica2] Registered new searcher Searcher@178c038 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1330819 T4737 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1330821 T4737 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=39, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=13.875, floorSegmentMB=1.19140625, forceMergeDeletesPctAllowed=28.19049544242447, segmentsPerTier=39.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.36574365566423817
   [junit4]   2> 1330822 T4737 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@11fb92b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@18e4db5),segFN=segments_1,generation=1}
   [junit4]   2> 1330822 T4737 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1330823 T4769 oasc.CoreContainer.registerCore registering core: delDeadColl_shard2_replica2
   [junit4]   2> 1330823 T4737 oass.SolrIndexSearcher.<init> Opening Searcher@1d1d6d3 main
   [junit4]   2> 1330824 T4795 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330825 T4795 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1330826 T4795 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1330827 T4769 oasc.ZkController.register Register replica - core:delDeadColl_shard2_replica2 address:http://127.0.0.1:56204/jm/af collection:delDeadColl shard:shard2
   [junit4]   2> 1330828 T4813 oasc.SolrCore.registerSearcher [delDeadColl_shard1_replica1] Registered new searcher Searcher@1d1d6d3 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1330829 T4769 oascc.SolrZkClient.makePath makePath: /collections/delDeadColl/leader_elect/shard2/election
   [junit4]   2> 1330832 T4737 oasc.CoreContainer.registerCore registering core: delDeadColl_shard1_replica1
   [junit4]   2> 1330833 T4737 oasc.ZkController.register Register replica - core:delDeadColl_shard1_replica1 address:http://127.0.0.1:51419/jm/af collection:delDeadColl shard:shard1
   [junit4]   2> 1330835 T4795 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1330839 T4769 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1330840 T4795 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1330843 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1330843 T4769 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=180000
   [junit4]   2> 1330844 T4742 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1330850 T4741 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1330858 T4795 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330864 T4795 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1330869 T4795 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330871 T4795 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1330871 T4795 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1330871 T4795 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1330873 T4795 oass.OpenExchangeRatesOrgPr

[...truncated too long message...]

tchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550494 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550494 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550494 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550494 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550495 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550495 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550495 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550495 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550495 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550495 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550495 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550496 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550496 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550496 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550496 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550496 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550496 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550496 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550497 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550497 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550497 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550497 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550497 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550497 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550497 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550498 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550498 T4759 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1550498 T4759 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene46: {}, docValues:{}, sim=DefaultSimilarity, locale=sl_SI, timezone=America/Yakutat
   [junit4]   2> NOTE: Linux 3.8.0-33-generic i386/Oracle Corporation 1.7.0_45 (32-bit)/cpus=8,threads=1,free=51084080,total=326107136
   [junit4]   2> NOTE: All tests run in this JVM: [TestSimpleQParserPlugin, TestStressReorder, HardAutoCommitTest, FieldAnalysisRequestHandlerTest, TestFastWriter, PingRequestHandlerTest, TestFuzzyAnalyzedSuggestions, TestCSVResponseWriter, TestFieldCollectionResource, SimplePostToolTest, TestSerializedLuceneMatchVersion, TestDistributedGrouping, SolrXmlInZkTest, TestIBSimilarityFactory, TestReload, TestFoldingMultitermQuery, TestJmxIntegration, BasicZkTest, TestIndexingPerformance, TestPartialUpdateDeduplication, TestDefaultSearchFieldResource, TestComponentsName, DirectSolrSpellCheckerTest, TestDocBasedVersionConstraints, DebugComponentTest, TestCodecSupport, TestNumberUtils, TestSchemaResource, TestStressRecovery, TestDynamicFieldResource, IndexSchemaRuntimeFieldTest, TestDistributedSearch, TestDistribDocBasedVersion, TestPhraseSuggestions, TestZkChroot, TestSolr4Spatial, TestUpdate, TestIndexSearcher, PrimUtilsTest, TestDFRSimilarityFactory, TestCloudManagedSchema, HdfsCollectionsAPIDistributedZkTest, TestSchemaVersionResource, RAMDirectoryFactoryTest, TestQuerySenderNoQuery, PluginInfoTest, SpellCheckCollatorTest, ChaosMonkeySafeLeaderTest, TestCollapseQParserPlugin, HighlighterTest, TestCollationKeyRangeQueries, PreAnalyzedFieldTest, RequestHandlersTest, TestAnalyzedSuggestions, NotRequiredUniqueKeyTest, FullSolrCloudDistribCmdsTest, TestRTGBase, TestDefaultSimilarityFactory, SpellCheckComponentTest, ChaosMonkeyNothingIsSafeTest, TestSolrCoreProperties, DirectUpdateHandlerTest, CurrencyFieldXmlFileTest, TestRandomMergePolicy, FileUtilsTest, TestRandomFaceting, DateFieldTest, TestRangeQuery, ConvertedLegacyTest, TestXIncludeConfig, TestAddFieldRealTimeGet, TestArbitraryIndexDir, OpenCloseCoreStressTest, AliasIntegrationTest, DefaultValueUpdateProcessorTest, TestFieldTypeResource, AlternateDirectoryTest, NumericFieldsTest, ChangedSchemaMergeTest, TestRecovery, TestLazyCores, MinimalSchemaTest, DirectSolrConnectionTest, TestAtomicUpdateErrorCases, CollectionsAPIDistributedZkTest, UnloadDistributedZkTest, OverseerTest, LeaderElectionIntegrationTest, AutoCommitTest, TestHashPartitioner, TestMultiCoreConfBootstrap, TestStressVersions, DistributedTermsComponentTest, StatsComponentTest, TestGroupingSearch, SolrCmdDistributorTest, PeerSyncTest, TestFiltering, SuggesterFSTTest, SuggesterTSTTest, SuggesterTest, SuggesterWFSTTest, TestCSVLoader, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestWordDelimiterFilterFactory, StatelessScriptUpdateProcessorFactoryTest, TestRemoteStreaming, DistanceFunctionTest, XsltUpdateRequestHandlerTest, DisMaxRequestHandlerTest, StandardRequestHandlerTest, DocumentAnalysisRequestHandlerTest, TestOmitPositions, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, LoggingHandlerTest, TestCollationField, UpdateRequestProcessorFactoryTest, UniqFieldsUpdateProcessorFactoryTest, CSVRequestHandlerTest, CoreContainerCoreInitFailuresTest, TestLFUCache, SOLR749Test, UpdateParamsTest, SolrIndexConfigTest, TestMergePolicyConfig, MultiTermTest, SampleTest, TestElisionMultitermQuery, SpellPossibilityIteratorTest, TestSweetSpotSimilarityFactory, TestLMDirichletSimilarityFactory, TestJmxMonitoredMap, TestFastOutputStream, URLClassifyProcessorTest, DateMathParserTest, SpellingQueryConverterTest, TestUtils, ZkNodePropsTest, CircularListTest, SolrTestCaseJ4Test, DeleteInactiveReplicaTest]
   [junit4] Completed on J1 in 227.44s, 1 test, 1 error <<< FAILURES!

[...truncated 130 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:426: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:406: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:489: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1276: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:912: There were test failures: 341 suites, 1498 tests, 1 error, 34 ignored (6 assumptions)

Total time: 53 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.7.0_45 -client -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure