You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/14 21:56:42 UTC

[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 732 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/732/
Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseSerialGC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.CustomCollectionTest.testDistribSearch

Error Message:
collection already exists: implicitcoll0

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: implicitcoll0
	at __randomizedtesting.SeedInfo.seed([9BA08767B02494A6:1A46097FC77BF49A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1537)
	at org.apache.solr.cloud.CustomCollectionTest.testCustomCollectionsAPI(CustomCollectionTest.java:192)
	at org.apache.solr.cloud.CustomCollectionTest.doTest(CustomCollectionTest.java:136)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication

Error Message:
[/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508949047/slave/collection1/data/index.20130815033555013, /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508949047/slave/collection1/data/index, /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508949047/slave/collection1/data] expected:<2> but was:<3>

Stack Trace:
java.lang.AssertionError: [/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508949047/slave/collection1/data/index.20130815033555013, /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508949047/slave/collection1/data/index, /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508949047/slave/collection1/data] expected:<2> but was:<3>
	at __randomizedtesting.SeedInfo.seed([9BA08767B02494A6:8FE8DC32932329B8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.handler.TestReplicationHandler.checkForSingleIndex(TestReplicationHandler.java:861)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigReplication(TestReplicationHandler.java:567)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 17560 lines...]
   [junit4] Suite: org.apache.solr.cloud.CustomCollectionTest
   [junit4]   2> 28369 T19 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /h_orw/a
   [junit4]   2> 28393 T19 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-CustomCollectionTest-1376505753554
   [junit4]   2> 28428 T19 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 28439 T20 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 28757 T19 oasc.ZkTestServer.run start zk server on port:49224
   [junit4]   2> 29209 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29383 T21 oazs.NIOServerCnxn.doIO WARN Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 30844 T25 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@336e48b3 name:ZooKeeperConnection Watcher:127.0.0.1:49224 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 30846 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 30855 T19 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 31033 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 31039 T28 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3aef8ec6 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 31040 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 31071 T19 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 31101 T19 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 31114 T19 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 31126 T19 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 31145 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 31187 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 31213 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 31215 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 31234 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 31236 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 31251 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 31253 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 31267 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 31269 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 31294 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 31296 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 31311 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 31312 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 31324 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 31325 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 31337 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 31340 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 31358 T19 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 31360 T19 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 32924 T19 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 33538 T19 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49228
   [junit4]   2> 33646 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 33649 T19 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 33650 T19 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533
   [junit4]   2> 33651 T19 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/'
   [junit4]   2> 33709 T19 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/solr.xml
   [junit4]   2> 33998 T19 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 34011 T19 oasc.CoreContainer.<init> New CoreContainer 2052953862
   [junit4]   2> 34012 T19 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/]
   [junit4]   2> 34091 T19 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 34093 T19 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 34093 T19 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 34094 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 34097 T19 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 34098 T19 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 34099 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 34100 T19 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 34101 T19 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 34102 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 34136 T19 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49224/solr
   [junit4]   2> 34178 T19 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 34183 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 34202 T40 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d04148c name:ZooKeeperConnection Watcher:127.0.0.1:49224 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 34204 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 34229 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 34254 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 34262 T42 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f961d73 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 34263 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 34314 T19 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 34346 T19 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 34394 T19 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 34405 T19 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49228_h_orw%2Fa
   [junit4]   2> 34427 T19 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49228_h_orw%2Fa
   [junit4]   2> 34441 T19 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 34487 T19 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 34506 T19 oasc.Overseer.start Overseer (id=90210681108824067-127.0.0.1:49228_h_orw%2Fa-n_0000000000) starting
   [junit4]   2> 34664 T19 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 34700 T44 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 34704 T19 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 34731 T19 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 34748 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 34800 T19 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533
   [junit4]   2> 34806 T43 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 34967 T19 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/conf/
   [junit4]   2> 34979 T19 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 34988 T19 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 34989 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 35076 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 35145 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 35170 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 35190 T47 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@193051e4 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 35191 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 35200 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 35225 T19 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 35738 T19 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 35747 T19 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49232
   [junit4]   2> 35748 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 35749 T19 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 35756 T19 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385
   [junit4]   2> 35757 T19 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/'
   [junit4]   2> 35814 T19 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/solr.xml
   [junit4]   2> 35958 T19 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 35961 T19 oasc.CoreContainer.<init> New CoreContainer 672058440
   [junit4]   2> 35962 T19 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/]
   [junit4]   2> 35971 T19 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 35972 T19 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 35974 T19 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 35975 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 35975 T19 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 35976 T19 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 35977 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 35978 T19 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 35979 T19 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 35980 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 36037 T19 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49224/solr
   [junit4]   2> 36039 T19 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 36043 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36054 T58 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b881d7e name:ZooKeeperConnection Watcher:127.0.0.1:49224 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36055 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36071 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 36101 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 36106 T60 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4cde17e5 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 36107 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 36127 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 37149 T19 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49232_h_orw%2Fa
   [junit4]   2> 37152 T19 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49232_h_orw%2Fa
   [junit4]   2> 37204 T42 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 37219 T60 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 37219 T47 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 37251 T19 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385
   [junit4]   2> 37397 T19 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/conf/
   [junit4]   2> 37404 T19 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 37405 T19 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 37406 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 37407 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 38146 T19 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 38165 T19 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49235
   [junit4]   2> 38171 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 38173 T19 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 38174 T19 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594
   [junit4]   2> 38176 T19 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/'
   [junit4]   2> 38281 T19 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/solr.xml
   [junit4]   2> 38548 T19 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 38551 T19 oasc.CoreContainer.<init> New CoreContainer 370722858
   [junit4]   2> 38552 T19 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/]
   [junit4]   2> 38558 T19 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 38559 T19 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 38561 T19 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 38567 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 38573 T19 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 38575 T19 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 38576 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 38577 T19 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 38580 T19 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 38581 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 38625 T19 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49224/solr
   [junit4]   2> 38626 T19 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 38636 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 38659 T72 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@737b6ad4 name:ZooKeeperConnection Watcher:127.0.0.1:49224 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 38666 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 38675 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 38704 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 38712 T74 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67afe177 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 38713 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 38746 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 39769 T19 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49235_h_orw%2Fa
   [junit4]   2> 39774 T19 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49235_h_orw%2Fa
   [junit4]   2> 39796 T47 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 39797 T60 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 39798 T74 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 39798 T42 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 39832 T19 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594
   [junit4]   2> 40061 T19 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/conf/
   [junit4]   2> 40071 T19 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 40072 T19 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 40073 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 40074 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 41303 T19 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 41339 T19 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49238
   [junit4]   2> 41342 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 41343 T19 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 41344 T19 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279
   [junit4]   2> 41346 T19 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/'
   [junit4]   2> 41444 T19 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/solr.xml
   [junit4]   2> 41775 T19 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 41782 T19 oasc.CoreContainer.<init> New CoreContainer 1249964669
   [junit4]   2> 41782 T19 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/]
   [junit4]   2> 41796 T19 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 41798 T19 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 41799 T19 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 41800 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 41801 T19 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 41804 T19 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 41805 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 41806 T19 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 41807 T19 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 41812 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 41853 T19 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49224/solr
   [junit4]   2> 41865 T19 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 41881 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 41885 T86 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39e026c9 name:ZooKeeperConnection Watcher:127.0.0.1:49224 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 41885 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 41900 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 41930 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 41941 T88 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a6c0d49 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 41943 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 41979 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 43034 T19 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49238_h_orw%2Fa
   [junit4]   2> 43038 T19 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49238_h_orw%2Fa
   [junit4]   2> 43056 T60 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 43057 T74 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 43058 T88 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 43059 T42 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 43062 T47 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 43087 T19 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279
   [junit4]   2> 43376 T19 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/conf/
   [junit4]   2> 43409 T19 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 43411 T19 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 43413 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 43416 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 44614 T19 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 44661 T19 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49241
   [junit4]   2> 44664 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 44671 T19 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 44673 T19 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630
   [junit4]   2> 44678 T19 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/'
   [junit4]   2> 44793 T19 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/solr.xml
   [junit4]   2> 45216 T19 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 45217 T19 oasc.CoreContainer.<init> New CoreContainer 1934585825
   [junit4]   2> 45218 T19 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/]
   [junit4]   2> 45235 T19 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 45238 T19 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 45239 T19 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 45241 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 45242 T19 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 45248 T19 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 45250 T19 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 45251 T19 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 45251 T19 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 45253 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 45375 T19 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49224/solr
   [junit4]   2> 45377 T19 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 45400 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 45425 T100 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@42d0a46b name:ZooKeeperConnection Watcher:127.0.0.1:49224 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 45426 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 45484 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 45528 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 45534 T102 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b29c61f name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 45545 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 45597 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 46640 T19 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49241_h_orw%2Fa
   [junit4]   2> 46652 T19 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49241_h_orw%2Fa
   [junit4]   2> 46678 T42 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 46680 T47 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 46682 T88 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 46683 T74 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 46682 T60 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 46686 T102 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 46750 T19 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630
   [junit4]   2> 47021 T19 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/conf/
   [junit4]   2> 47036 T19 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 47039 T19 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 47041 T19 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 47043 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 47088 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 47102 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47107 T105 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@453d41a7 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47115 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47127 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 47153 T19 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 47183 T19 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47191 T107 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3500b5d7 name:ZooKeeperConnection Watcher:127.0.0.1:49224/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47193 T19 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47233 T19 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 50315 T93 oasha.CollectionsHandler.handleCreateAction Creating Collection : shards=a,b,c&maxShardsPerNode=4&router=implicit&name=implicitcoll0&replicationFactor=4&action=CREATE&wt=javabin&version=2
   [junit4]   2> 50330 T42 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 50349 T44 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "shards":"a,b,c",
   [junit4]   2> 	  "maxShardsPerNode":"4",
   [junit4]   2> 	  "router":"implicit",
   [junit4]   2> 	  "name":"implicitcoll0",
   [junit4]   2> 	  "replicationFactor":"4"}
   [junit4]   2> 50352 T44 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "shards":"a,b,c",
   [junit4]   2> 	  "maxShardsPerNode":"4",
   [junit4]   2> 	  "router":"implicit",
   [junit4]   2> 	  "name":"implicitcoll0",
   [junit4]   2> 	  "replicationFactor":"4"}
   [junit4]   2> 51508 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 51520 T43 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: implicitcoll0
   [junit4]   2> 51521 T43 oasc.Overseer$ClusterStateUpdater.createCollection Create collection implicitcoll0 with shards [a, b, c]
   [junit4]   2> 51643 T88 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51644 T102 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51646 T60 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51647 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51649 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51648 T105 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51648 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51647 T47 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 51798 T44 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [a, b, c] , repFactor : 4
   [junit4]   2> 51800 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_a_replica1 as part of slice a of collection implicitcoll0 on 127.0.0.1:49241_h_orw%2Fa
   [junit4]   2> 51866 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_a_replica2 as part of slice a of collection implicitcoll0 on 127.0.0.1:49235_h_orw%2Fa
   [junit4]   2> 51874 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_a_replica3 as part of slice a of collection implicitcoll0 on 127.0.0.1:49232_h_orw%2Fa
   [junit4]   2> 51878 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_a_replica4 as part of slice a of collection implicitcoll0 on 127.0.0.1:49238_h_orw%2Fa
   [junit4]   2> 51898 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_b_replica1 as part of slice b of collection implicitcoll0 on 127.0.0.1:49228_h_orw%2Fa
   [junit4]   2> 51901 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_b_replica2 as part of slice b of collection implicitcoll0 on 127.0.0.1:49241_h_orw%2Fa
   [junit4]   2> 51917 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_b_replica3 as part of slice b of collection implicitcoll0 on 127.0.0.1:49235_h_orw%2Fa
   [junit4]   2> 51920 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_b_replica4 as part of slice b of collection implicitcoll0 on 127.0.0.1:49232_h_orw%2Fa
   [junit4]   2> 51924 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_c_replica1 as part of slice c of collection implicitcoll0 on 127.0.0.1:49238_h_orw%2Fa
   [junit4]   2> 51947 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_c_replica2 as part of slice c of collection implicitcoll0 on 127.0.0.1:49228_h_orw%2Fa
   [junit4]   2> 51952 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_c_replica3 as part of slice c of collection implicitcoll0 on 127.0.0.1:49241_h_orw%2Fa
   [junit4]   2> 51955 T44 oasc.OverseerCollectionProcessor.createCollection Creating shard implicitcoll0_c_replica4 as part of slice c of collection implicitcoll0 on 127.0.0.1:49235_h_orw%2Fa
   [junit4]   2> 52096 T70 oasc.ZkController.publish publishing core=implicitcoll0_b_replica3 state=down
   [junit4]   2> 52115 T66 oasc.ZkController.publish publishing core=implicitcoll0_a_replica2 state=down
   [junit4]   2> 52115 T55 oasc.ZkController.publish publishing core=implicitcoll0_b_replica4 state=down
   [junit4]   2> 52115 T95 oasc.ZkController.publish publishing core=implicitcoll0_a_replica1 state=down
   [junit4]   2> 52115 T54 oasc.ZkController.publish publishing core=implicitcoll0_a_replica3 state=down
   [junit4]   2> 52113 T81 oasc.ZkController.publish publishing core=implicitcoll0_a_replica4 state=down
   [junit4]   2> 52100 T80 oasc.ZkController.publish publishing core=implicitcoll0_c_replica1 state=down
   [junit4]   2> 52129 T70 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52100 T94 oasc.ZkController.publish publishing core=implicitcoll0_b_replica2 state=down
   [junit4]   2> 52100 T35 oasc.ZkController.publish publishing core=implicitcoll0_c_replica2 state=down
   [junit4]   2> 52099 T69 oasc.ZkController.publish publishing core=implicitcoll0_c_replica4 state=down
   [junit4]   2> 52099 T36 oasc.ZkController.publish publishing core=implicitcoll0_b_replica1 state=down
   [junit4]   2> 52129 T66 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52142 T95 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52144 T81 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52145 T80 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52147 T54 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52150 T35 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52117 T98 oasc.ZkController.publish publishing core=implicitcoll0_c_replica3 state=down
   [junit4]   2> 52151 T36 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52150 T94 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52147 T69 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52145 T55 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 52159 T98 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 53233 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 53253 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"b",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_b_replica3",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49235_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49235/h_orw/a"}
   [junit4]   2> 53316 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"a",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_a_replica2",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49235_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49235/h_orw/a"}
   [junit4]   2> 53355 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"a",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_a_replica1",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49241_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49241/h_orw/a"}
   [junit4]   2> 53429 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"b",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_b_replica4",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49232_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49232/h_orw/a"}
   [junit4]   2> 53470 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node5",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"a",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_a_replica3",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49232_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49232/h_orw/a"}
   [junit4]   2> 53500 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node6",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"a",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_a_replica4",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49238_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49238/h_orw/a"}
   [junit4]   2> 53538 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node7",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"c",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_c_replica1",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49238_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49238/h_orw/a"}
   [junit4]   2> 53588 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node8",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"c",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_c_replica4",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49235_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49235/h_orw/a"}
   [junit4]   2> 53611 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node9",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"c",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_c_replica2",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49228_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49228/h_orw/a"}
   [junit4]   2> 53654 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node10",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"b",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_b_replica2",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49241_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49241/h_orw/a"}
   [junit4]   2> 53705 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node11",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"b",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_b_replica1",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49228_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49228/h_orw/a"}
   [junit4]   2> 53751 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node12",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"c",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"implicitcoll0_c_replica3",
   [junit4]   2> 	  "collection":"implicitcoll0",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49241_h_orw%2Fa",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49241/h_orw/a"}
   [junit4]   2> 53800 T102 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 53801 T47 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 53802 T105 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 53803 T107 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 53803 T74 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 53805 T88 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 53803 T60 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 53840 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 54132 T70 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_b_replica3
   [junit4]   2> 54135 T70 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_b_replica3/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_b_replica3/core.properties (No such file or directory)
   [junit4]   2> 54140 T70 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_b_replica3' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_b_replica3
   [junit4]   2> 54142 T70 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54142 T66 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_a_replica2
   [junit4]   2> 54145 T66 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_a_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_a_replica2/core.properties (No such file or directory)
   [junit4]   2> 54147 T95 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_a_replica1
   [junit4]   2> 54147 T81 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_a_replica4
   [junit4]   2> 54149 T95 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_a_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_a_replica1/core.properties (No such file or directory)
   [junit4]   2> 54151 T95 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_a_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_a_replica1
   [junit4]   2> 54151 T95 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54147 T66 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_a_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_a_replica2
   [junit4]   2> 54153 T66 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54152 T54 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_a_replica3
   [junit4]   2> 54150 T80 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_c_replica1
   [junit4]   2> 54157 T54 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_a_replica3/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_a_replica3/core.properties (No such file or directory)
   [junit4]   2> 54149 T81 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_a_replica4/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_a_replica4/core.properties (No such file or directory)
   [junit4]   2> 54160 T81 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_a_replica4' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_a_replica4
   [junit4]   2> 54161 T81 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54163 T70 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54159 T54 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_a_replica3' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_a_replica3
   [junit4]   2> 54158 T94 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_b_replica2
   [junit4]   2> 54158 T80 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_c_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_c_replica1/core.properties (No such file or directory)
   [junit4]   2> 54170 T80 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_c_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_c_replica1
   [junit4]   2> 54172 T80 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54169 T54 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54169 T94 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_b_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_b_replica2/core.properties (No such file or directory)
   [junit4]   2> 54175 T94 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_b_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_b_replica2
   [junit4]   2> 54176 T94 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54167 T70 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54166 T98 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_c_replica3
   [junit4]   2> 54189 T98 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_c_replica3/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_c_replica3/core.properties (No such file or directory)
   [junit4]   2> 54190 T80 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54191 T80 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54164 T66 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54193 T66 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54164 T95 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54163 T55 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_b_replica4
   [junit4]   2> 54197 T94 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54161 T69 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_c_replica4
   [junit4]   2> 54198 T94 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54198 T55 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_b_replica4/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_b_replica4/core.properties (No such file or directory)
   [junit4]   2> 54202 T55 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_b_replica4' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_b_replica4
   [junit4]   2> 54202 T55 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54196 T95 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54207 T70 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54194 T54 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54190 T98 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_c_replica3' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_c_replica3
   [junit4]   2> 54213 T98 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54172 T81 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54220 T94 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54210 T70 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54214 T54 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54213 T66 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54211 T80 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54209 T55 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54200 T69 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_c_replica4/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_c_replica4/core.properties (No such file or directory)
   [junit4]   2> 54229 T69 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_c_replica4' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_c_replica4
   [junit4]   2> 54230 T69 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 54228 T55 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54227 T80 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54222 T94 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54235 T98 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54236 T98 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54221 T95 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54240 T95 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54246 T69 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:implicitcoll0
   [junit4]   2> 54247 T69 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54221 T81 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 54252 T54 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54227 T66 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54254 T54 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54261 T55 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54262 T55 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54281 T98 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54282 T69 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54283 T81 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 54283 T69 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54282 T98 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54284 T81 oascc.SolrZkClient.makePath makePath: /collections/implicitcoll0
   [junit4]   2> 54286 T70 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54292 T80 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54297 T94 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54316 T54 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54320 T66 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54323 T95 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54325 T55 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54337 T70 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_b_replica3/'
   [junit4]   2> 54337 T80 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_c_replica1/'
   [junit4]   2> 54354 T81 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54355 T98 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54355 T55 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_b_replica4/'
   [junit4]   2> 54355 T54 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty1-1376505760385/implicitcoll0_a_replica3/'
   [junit4]   2> 54358 T94 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_b_replica2/'
   [junit4]   2> 54364 T95 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_a_replica1/'
   [junit4]   2> 54366 T66 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_a_replica2/'
   [junit4]   2> 54367 T69 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 54378 T98 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty4-1376505768630/implicitcoll0_c_replica3/'
   [junit4]   2> 54380 T81 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty3-1376505765279/implicitcoll0_a_replica4/'
   [junit4]   2> 54399 T69 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-jetty2-1376505762594/implicitcoll0_c_replica4/'
   [junit4]   2> 55153 T35 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_c_replica2
   [junit4]   2> 55159 T36 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for implicitcoll0_b_replica1
   [junit4]   2> 55194 T36 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_b_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_b_replica1/core.properties (No such file or directory)
   [junit4]   2> 55195 T36 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_b_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_b_replica1
   [junit4]   2> 55196 T36 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 55201 T36 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 55202 T36 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 55215 T36 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_b_replica1/'
   [junit4]   2> 55232 T35 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_c_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_c_replica2/core.properties (No such file or directory)
   [junit4]   2> 55241 T35 oasc.CoreContainer.create Creating SolrCore 'implicitcoll0_c_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_c_replica2
   [junit4]   2> 55243 T35 oasc.ZkController.createCollectionZkNode Check for collection zkNode:implicitcoll0
   [junit4]   2> 55249 T35 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 55251 T35 oasc.ZkController.readConfigName Load collection config from:/collections/implicitcoll0
   [junit4]   2> 55279 T35 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CustomCollectionTest-controljetty-1376505756533/implicitcoll0_c_replica2/'
   [junit4]   2> 56256 T95 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56265 T69 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56295 T55 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56323 T80 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56414 T81 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56469 T98 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56524 T94 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56606 T70 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56659 T54 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56662 T66 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 56818 T36 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 57312 T35 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 57521 T69 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 57521 T95 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 57550 T95 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 57627 T69 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 57724 T81 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 57775 T81 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
  

[...truncated too long message...]

estReplicationHandler$SolrInstance-1376508988434/master/collection1/data/index.20130815033635622;done=false>>]
   [junit4]   2> 3270794 T5558 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508988434/master/collection1/data/index.20130815033635622
   [junit4]   2> 3270799 T5558 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508988434/master/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508988434/master/collection1/data;done=false>>]
   [junit4]   2> 3270800 T5558 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508988434/master/collection1/data
   [junit4]   2> 3270801 T5558 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 3270875 T5558 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=216950896
   [junit4]   2> 3270877 T5558 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@38256092
   [junit4]   2> 3270878 T5558 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=3,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0}
   [junit4]   2> 3270879 T5558 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 3270880 T5558 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 3270881 T5558 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 3270883 T5558 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 3270884 T5558 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 3270885 T5558 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508989355/slave/collection1/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508989355/slave/collection1/data;done=false>>]
   [junit4]   2> 3270886 T5558 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508989355/slave/collection1/data
   [junit4]   2> 3270887 T5558 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508989355/slave/collection1/data/index.20130815033635126 [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508989355/slave/collection1/data/index.20130815033635126;done=false>>]
   [junit4]   2> 3270887 T5558 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.handler.TestReplicationHandler$SolrInstance-1376508989355/slave/collection1/data/index.20130815033635126
   [junit4]   2> 3270892 T5558 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 3270975 T5558 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=HighCompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=499), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=HIGH_COMPRESSION, chunkSize=499)), sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=en_GB, timezone=Australia/Perth
   [junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=2,threads=1,free=101405904,total=268877824
   [junit4]   2> NOTE: All tests run in this JVM: [TestNRTOpen, CustomCollectionTest, TestFieldResource, SolrIndexConfigTest, RAMDirectoryFactoryTest, ScriptEngineTest, TestLuceneMatchVersion, UpdateRequestProcessorFactoryTest, BasicZkTest, HdfsDirectoryTest, TestFiltering, TestManagedSchema, OpenCloseCoreStressTest, ConvertedLegacyTest, TestSearchPerf, TestInfoStreamLogging, MBeansHandlerTest, HdfsUnloadDistributedZkTest, BlockDirectoryTest, TestCloudManagedSchema, TestFoldingMultitermQuery, CoreContainerCoreInitFailuresTest, TestSerializedLuceneMatchVersion, TestStressLucene, CopyFieldTest, TestSystemIdResolver, TestRealTimeGet, SliceStateTest, HdfsBasicDistributedZk2Test, ReturnFieldsTest, TestPartialUpdateDeduplication, AssignTest, WordBreakSolrSpellCheckerTest, NumericFieldsTest, TestFaceting, PrimitiveFieldTypeTest, PingRequestHandlerTest, BadCopyFieldTest, TestIndexingPerformance, TestSchemaVersionResource, AlternateDirectoryTest, MoreLikeThisHandlerTest, TestSchemaNameResource, TestDynamicFieldCollectionResource, AnalysisAfterCoreReloadTest, TestStressReorder, CoreAdminHandlerTest, OverseerTest, HdfsSyncSliceTest, TestCloudManagedSchemaAddField, TestSurroundQueryParser, TestJoin, MinimalSchemaTest, SystemInfoHandlerTest, PathHierarchyTokenizerFactoryTest, TestQuerySenderNoQuery, TestDefaultSimilarityFactory, TestJmxMonitoredMap, TestUniqueKeyFieldResource, CircularListTest, TestSolrIndexConfig, TestClassNameShortening, SOLR749Test, XmlUpdateRequestHandlerTest, QueryParsingTest, XsltUpdateRequestHandlerTest, PluginInfoTest, TestStressVersions, TestDocSet, BasicDistributedZkTest, DefaultValueUpdateProcessorTest, RegexBoostProcessorTest, TestLMDirichletSimilarityFactory, DistanceFunctionTest, FieldMutatingUpdateProcessorTest, TestLMJelinekMercerSimilarityFactory, TestSweetSpotSimilarityFactory, FieldAnalysisRequestHandlerTest, TestSchemaSimilarityResource, TestCoreContainer, TestRemoteStreaming, TestPerFieldSimilarity, ShardSplitTest, IndexSchemaTest, NotRequiredUniqueKeyTest, ChaosMonkeySafeLeaderTest, TestRangeQuery, TestDistributedSearch, TestUtils, BinaryUpdateRequestHandlerTest, TestCoreDiscovery, PreAnalyzedFieldTest, DocumentAnalysisRequestHandlerTest, SolrCoreCheckLockOnStartupTest, SpellingQueryConverterTest, TestCSVLoader, TestCopyFieldCollectionResource, UUIDFieldTest, TestArbitraryIndexDir, OutputWriterTest, TestWordDelimiterFilterFactory, TestGroupingSearch, AliasIntegrationTest, FastVectorHighlighterTest, SolrCoreTest, PreAnalyzedUpdateProcessorTest, TestStressRecovery, CollectionsAPIDistributedZkTest, TestBinaryResponseWriter, DocValuesMultiTest, TestJmxIntegration, SpellCheckCollatorTest, SuggesterTSTTest, ZkSolrClientTest, LeaderElectionIntegrationTest, HardAutoCommitTest, TestSolrXMLSerializer, SignatureUpdateProcessorFactoryTest, TestSolrQueryParserDefaultOperatorResource, BasicFunctionalityTest, ChaosMonkeyNothingIsSafeTest, DirectSolrConnectionTest, BasicDistributedZk2Test, TestUpdate, DistributedSpellCheckComponentTest, FileBasedSpellCheckerTest, TestPHPSerializedResponseWriter, LukeRequestHandlerTest, TestFuzzyAnalyzedSuggestions, SuggesterTest, TestCSVResponseWriter, TestBadConfig, DistributedQueryElevationComponentTest, TestComponentsName, FileUtilsTest, TestDFRSimilarityFactory, AddSchemaFieldsUpdateProcessorFactoryTest, SpellPossibilityIteratorTest, TestQueryTypes, TestFastWriter, RequiredFieldsTest, ZkControllerTest, TestConfig, SortByFunctionTest, TestManagedSchemaFieldResource, DateFieldTest, TestDefaultSearchFieldResource, CachingDirectoryFactoryTest, TestSolrXml, LegacyHTMLStripCharFilterTest, HdfsCollectionsAPIDistributedZkTest, StressHdfsTest, TestCharFilters, UnloadDistributedZkTest, SyncSliceTest, ShardRoutingTest, RecoveryZkTest, TestReplicationHandler]
   [junit4] Completed in 76.67s, 14 tests, 1 failure <<< FAILURES!

[...truncated 477 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:389: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:369: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:450: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1230: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:873: There were test failures: 320 suites, 1361 tests, 1 error, 1 failure, 40 ignored (5 assumptions)

Total time: 117 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure