You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/11 07:49:19 UTC

[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 720 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/720/
Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
collection already exists: awholynewcollection_0

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_0
	at __randomizedtesting.SeedInfo.seed([7E7F9185C22CDD1B:FF991F9DB573BD27]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1537)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1551)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:439)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:147)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 9762 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 2321491 T4368 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /yc_/
   [junit4]   2> 2321503 T4368 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1376197614928
   [junit4]   2> 2321505 T4368 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 2321507 T4369 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2321614 T4368 oasc.ZkTestServer.run start zk server on port:50294
   [junit4]   2> 2321617 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2321627 T4375 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c102e14 name:ZooKeeperConnection Watcher:127.0.0.1:50294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2321627 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2321628 T4368 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2321646 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2321650 T4377 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64ccf994 name:ZooKeeperConnection Watcher:127.0.0.1:50294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2321650 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2321651 T4368 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2321670 T4368 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2321678 T4368 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2321684 T4368 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2321692 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2321693 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2321705 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2321707 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2321722 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2321723 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2321733 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2321734 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2321742 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2321744 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2321751 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2321752 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2321758 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2321759 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2321766 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2321768 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2321775 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2321777 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2321790 T4368 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2321792 T4368 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2322351 T4368 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2322367 T4368 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50297
   [junit4]   2> 2322368 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2322369 T4368 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2322369 T4368 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234
   [junit4]   2> 2322369 T4368 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/'
   [junit4]   2> 2322436 T4368 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/solr.xml
   [junit4]   2> 2322625 T4368 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2322626 T4368 oasc.CoreContainer.<init> New CoreContainer 562991259
   [junit4]   2> 2322626 T4368 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/]
   [junit4]   2> 2322631 T4368 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2322632 T4368 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2322632 T4368 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2322633 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2322633 T4368 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2322634 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2322635 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2322635 T4368 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2322636 T4368 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2322637 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2322664 T4368 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50294/solr
   [junit4]   2> 2322665 T4368 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2322667 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2322674 T4388 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b260dbd name:ZooKeeperConnection Watcher:127.0.0.1:50294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2322675 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2322682 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2322703 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2322709 T4390 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5bff0e51 name:ZooKeeperConnection Watcher:127.0.0.1:50294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2322709 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2322718 T4368 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2322730 T4368 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2322742 T4368 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2322748 T4368 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50297_yc_
   [junit4]   2> 2322750 T4368 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50297_yc_
   [junit4]   2> 2322758 T4368 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2322774 T4368 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2322783 T4368 oasc.Overseer.start Overseer (id=90190486892511235-127.0.0.1:50297_yc_-n_0000000000) starting
   [junit4]   2> 2322801 T4368 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2322813 T4392 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2322818 T4368 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2322830 T4368 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2322839 T4368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2322870 T4391 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2322878 T4368 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234
   [junit4]   2> 2323078 T4368 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/conf/
   [junit4]   2> 2323087 T4368 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2323089 T4368 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2323090 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2323091 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2323122 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 2323124 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2323128 T4395 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18baffd5 name:ZooKeeperConnection Watcher:127.0.0.1:50294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2323129 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2323134 T4368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2323142 T4368 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 2323651 T4368 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2323665 T4368 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50301
   [junit4]   2> 2323666 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2323667 T4368 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2323667 T4368 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376197616565
   [junit4]   2> 2323668 T4368 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376197616565/'
   [junit4]   2> 2323736 T4368 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376197616565/solr.xml
   [junit4]   2> 2323899 T4368 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2323900 T4368 oasc.CoreContainer.<init> New CoreContainer 748420946
   [junit4]   2> 2323900 T4368 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376197616565/]
   [junit4]   2> 2323903 T4368 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2323903 T4368 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2323904 T4368 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2323904 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2323905 T4368 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2323906 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2323907 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2323907 T4368 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2323907 T4368 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2323908 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2323941 T4368 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50294/solr
   [junit4]   2> 2323944 T4368 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2323946 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2323951 T4406 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5198744 name:ZooKeeperConnection Watcher:127.0.0.1:50294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2323952 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2323959 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2323968 T4370 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x1406bc51d590005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 2323982 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2323985 T4408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@781db88e name:ZooKeeperConnection Watcher:127.0.0.1:50294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2323985 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2324006 T4368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2325024 T4368 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50301_yc_
   [junit4]   2> 2325028 T4368 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50301_yc_
   [junit4]   2> 2325041 T4408 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2325041 T4390 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2325041 T4395 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2325054 T4368 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376197616565
   [junit4]   2> 2325260 T4368 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376197616565/conf/
   [junit4]   2> 2325271 T4368 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2325273 T4368 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2325274 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2325275 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2325884 T4368 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2325895 T4368 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50304
   [junit4]   2> 2325896 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2325897 T4368 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2325898 T4368 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791
   [junit4]   2> 2325899 T4368 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/'
   [junit4]   2> 2325971 T4368 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/solr.xml
   [junit4]   2> 2326204 T4368 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2326211 T4368 oasc.CoreContainer.<init> New CoreContainer 858064540
   [junit4]   2> 2326213 T4368 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/]
   [junit4]   2> 2326217 T4368 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2326218 T4368 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2326219 T4368 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2326220 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2326221 T4368 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2326222 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2326222 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2326223 T4368 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2326224 T4368 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2326225 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2326255 T4368 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50294/solr
   [junit4]   2> 2326257 T4368 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2326263 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2326266 T4420 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1afc9331 name:ZooKeeperConnection Watcher:127.0.0.1:50294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2326266 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2326276 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2326304 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2326310 T4422 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5db3ba1c name:ZooKeeperConnection Watcher:127.0.0.1:50294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2326311 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2326334 T4368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2327346 T4368 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50304_yc_
   [junit4]   2> 2327350 T4368 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50304_yc_
   [junit4]   2> 2327362 T4408 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2327363 T4422 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2327365 T4390 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2327367 T4395 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2327380 T4368 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791
   [junit4]   2> 2327558 T4368 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/conf/
   [junit4]   2> 2327566 T4368 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2327567 T4368 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2327568 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2327569 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2328237 T4368 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2328248 T4368 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50307
   [junit4]   2> 2328249 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2328251 T4368 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2328252 T4368 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023
   [junit4]   2> 2328253 T4368 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/'
   [junit4]   2> 2328309 T4368 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/solr.xml
   [junit4]   2> 2328476 T4368 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2328477 T4368 oasc.CoreContainer.<init> New CoreContainer 1087860551
   [junit4]   2> 2328478 T4368 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/]
   [junit4]   2> 2328480 T4368 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2328481 T4368 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2328481 T4368 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2328482 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2328482 T4368 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2328483 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2328483 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2328484 T4368 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2328485 T4368 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2328485 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2328512 T4368 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50294/solr
   [junit4]   2> 2328514 T4368 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2328520 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2328522 T4434 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1220a635 name:ZooKeeperConnection Watcher:127.0.0.1:50294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2328522 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2328530 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2328550 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2328554 T4436 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17dbd6f9 name:ZooKeeperConnection Watcher:127.0.0.1:50294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2328555 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2328576 T4368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2329587 T4368 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50307_yc_
   [junit4]   2> 2329590 T4368 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50307_yc_
   [junit4]   2> 2329598 T4422 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2329599 T4395 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2329604 T4408 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2329605 T4436 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2329608 T4390 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2329621 T4368 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023
   [junit4]   2> 2329826 T4368 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/conf/
   [junit4]   2> 2329835 T4368 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2329836 T4368 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2329837 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2329838 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2330406 T4368 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2330414 T4368 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50310
   [junit4]   2> 2330416 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2330416 T4368 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2330417 T4368 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286
   [junit4]   2> 2330418 T4368 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/'
   [junit4]   2> 2330488 T4368 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/solr.xml
   [junit4]   2> 2330661 T4368 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2330664 T4368 oasc.CoreContainer.<init> New CoreContainer 1278631338
   [junit4]   2> 2330666 T4368 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/]
   [junit4]   2> 2330671 T4368 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2330673 T4368 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2330673 T4368 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2330674 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2330675 T4368 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2330676 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2330677 T4368 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2330677 T4368 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2330678 T4368 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2330679 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2330701 T4368 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50294/solr
   [junit4]   2> 2330702 T4368 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2330707 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2330716 T4448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f1fbffe name:ZooKeeperConnection Watcher:127.0.0.1:50294 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2330716 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2330722 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2330746 T4368 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2330752 T4450 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@11529410 name:ZooKeeperConnection Watcher:127.0.0.1:50294/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2330752 T4368 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2330765 T4368 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2331776 T4368 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50310_yc_
   [junit4]   2> 2331779 T4368 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50310_yc_
   [junit4]   2> 2331791 T4408 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2331791 T4422 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2331798 T4450 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2331802 T4436 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2331805 T4395 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2331815 T4368 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286
   [junit4]   2> 2331822 T4390 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2331997 T4368 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/conf/
   [junit4]   2> 2332010 T4368 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2332012 T4368 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2332013 T4368 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2332014 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2332046 T4368 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2332075 T4398 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&name=nodes_used_collection&replicationFactor=2&action=CREATE&wt=javabin&version=2
   [junit4]   2> 2332079 T4390 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2332085 T4392 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 2332085 T4392 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 2332091 T4390 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2333544 T4391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2333545 T4391 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: nodes_used_collection
   [junit4]   2> 2333545 T4391 oasc.Overseer$ClusterStateUpdater.createCollection Create collection nodes_used_collection with shards [shard1, shard2]
   [junit4]   2> 2333554 T4450 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2333555 T4390 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2333555 T4395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2333560 T4436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2333563 T4422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2333567 T4408 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2333616 T4392 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 2333617 T4392 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:50304_yc_
   [junit4]   2> 2333618 T4392 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:50310_yc_
   [junit4]   2> 2333619 T4392 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:50297_yc_
   [junit4]   2> 2333621 T4392 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:50307_yc_
   [junit4]   2> 2333654 T4385 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down
   [junit4]   2> 2333668 T4385 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2333673 T4414 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down
   [junit4]   2> 2333675 T4427 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down
   [junit4]   2> 2333673 T4445 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down
   [junit4]   2> 2333679 T4427 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2333680 T4414 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2333683 T4445 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2335082 T4391 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2335084 T4391 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2335084 T4391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50297_yc_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50297/yc_"}
   [junit4]   2> 2335100 T4391 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2335101 T4391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50304_yc_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50304/yc_"}
   [junit4]   2> 2335113 T4391 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2335114 T4391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50307_yc_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50307/yc_"}
   [junit4]   2> 2335125 T4391 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2335126 T4391 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50310_yc_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50310/yc_"}
   [junit4]   2> 2335145 T4450 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2335145 T4436 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2335146 T4422 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2335146 T4408 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2335148 T4390 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2335150 T4395 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2335675 T4385 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 2335677 T4385 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/nodes_used_collection_shard2_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/nodes_used_collection_shard2_replica1/core.properties (No such file or directory)
   [junit4]   2> 2335678 T4385 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/nodes_used_collection_shard2_replica1
   [junit4]   2> 2335678 T4385 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2335681 T4385 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2335682 T4385 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2335683 T4427 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 2335684 T4427 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/nodes_used_collection_shard2_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/nodes_used_collection_shard2_replica2/core.properties (No such file or directory)
   [junit4]   2> 2335685 T4427 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/nodes_used_collection_shard2_replica2
   [junit4]   2> 2335686 T4427 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2335687 T4445 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 2335689 T4445 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/nodes_used_collection_shard1_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/nodes_used_collection_shard1_replica2/core.properties (No such file or directory)
   [junit4]   2> 2335689 T4445 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/nodes_used_collection_shard1_replica2
   [junit4]   2> 2335690 T4445 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2335691 T4414 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 2335693 T4414 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/nodes_used_collection_shard1_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/nodes_used_collection_shard1_replica1/core.properties (No such file or directory)
   [junit4]   2> 2335693 T4414 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/nodes_used_collection_shard1_replica1
   [junit4]   2> 2335694 T4414 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2335697 T4445 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2335697 T4445 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2335700 T4427 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2335701 T4427 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2335704 T4445 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 2335705 T4445 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2335707 T4427 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 2335707 T4427 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2335710 T4385 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 2335711 T4385 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2335712 T4414 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2335713 T4414 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2335720 T4385 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2335729 T4445 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2335731 T4414 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2335732 T4427 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2335733 T4385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 2335739 T4427 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 2335852 T4445 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 2335862 T4414 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2335867 T4414 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 2336139 T4414 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 2336235 T4427 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 2336275 T4385 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 2336292 T4445 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 2336451 T4427 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2336506 T4427 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2336529 T4427 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica2] Schema name=test
   [junit4]   2> 2336749 T4414 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2336758 T4385 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2336773 T4385 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2336790 T4414 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2336831 T4414 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica1] Schema name=test
   [junit4]   2> 2336836 T4385 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica1] Schema name=test
   [junit4]   2> 2336850 T4445 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2336943 T4445 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2336968 T4445 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica2] Schema name=test
   [junit4]   2> 2340660 T4427 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2340782 T4427 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2340803 T4427 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2340984 T4427 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2340993 T4427 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2341061 T4414 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2341070 T4427 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341087 T4414 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2341100 T4427 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341102 T4427 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341102 T4427 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341109 T4414 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2341115 T4427 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341116 T4427 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341117 T4427 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2341117 T4427 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/nodes_used_collection_shard2_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023//nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 2341120 T4427 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2daa38c7
   [junit4]   2> 2341139 T4427 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023//nodes_used_collection_shard2_replica2/data
   [junit4]   2> 2341140 T4427 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023//nodes_used_collection_shard2_replica2/data/index/
   [junit4]   2> 2341141 T4427 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023/nodes_used_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2341142 T4427 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376197621023//nodes_used_collection_shard2_replica2/data/index
   [junit4]   2> 2341235 T4385 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2341239 T4427 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@492cd14 lockFactory=org.apache.lucene.store.NativeFSLockFactory@1ef9efd1),segFN=segments_1,generation=1}
   [junit4]   2> 2341240 T4427 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2341279 T4427 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2341280 T4427 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2341281 T4427 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2341283 T4427 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2341284 T4427 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2341285 T4427 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2341286 T4427 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2341287 T4427 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2341288 T4427 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2341308 T4385 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2341328 T4414 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2341332 T4385 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2341321 T4427 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2341360 T4445 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2341374 T4414 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2341377 T4427 oass.SolrIndexSearcher.<init> Opening Searcher@1f760d55 main
   [junit4]   2> 2341379 T4427 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2341380 T4427 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2341400 T4445 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2341406 T4414 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341418 T4445 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2341426 T4414 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341430 T4414 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341430 T4414 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341425 T4427 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica2
   [junit4]   2> 2341420 T4456 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica2] Registered new searcher Searcher@1f760d55 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2341432 T4427 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica2 address:http://127.0.0.1:50307/yc_ collection:nodes_used_collection shard:shard2
   [junit4]   2> 2341433 T4427 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 2341444 T4414 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341445 T4414 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341446 T4414 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2341447 T4414 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/nodes_used_collection_shard1_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791//nodes_used_collection_shard1_replica1/data/
   [junit4]   2> 2341448 T4414 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2daa38c7
   [junit4]   2> 2341460 T4414 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791//nodes_used_collection_shard1_replica1/data
   [junit4]   2> 2341462 T4414 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791//nodes_used_collection_shard1_replica1/data/index/
   [junit4]   2> 2341462 T4414 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791/nodes_used_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2341464 T4414 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376197618791//nodes_used_collection_shard1_replica1/data/index
   [junit4]   2> 2341514 T4414 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@13563135 lockFactory=org.apache.lucene.store.NativeFSLockFactory@689eb62b),segFN=segments_1,generation=1}
   [junit4]   2> 2341515 T4414 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2341528 T4414 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2341528 T4414 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2341530 T4414 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2341531 T4414 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2341532 T4414 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2341532 T4414 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2341533 T4414 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2341534 T4414 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2341535 T4414 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2341550 T4414 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2341559 T4427 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard2/election
   [junit4]   2> 2341567 T4385 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2341578 T4414 oass.SolrIndexSearcher.<init> Opening Searcher@de4080 main
   [junit4]   2> 2341581 T4414 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2341582 T4414 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2341598 T4385 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2341649 T4445 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2341653 T4385 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341653 T4457 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica1] Registered new searcher Searcher@de4080 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2341661 T4414 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica1
   [junit4]   2> 2341662 T4414 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica1 address:http://127.0.0.1:50304/yc_ collection:nodes_used_collection shard:shard1
   [junit4]   2> 2341662 T4414 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 2341663 T4445 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2341674 T4445 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341675 T4385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341676 T4385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341677 T4385 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341676 T4427 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 2341684 T4445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341685 T4445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341689 T4445 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2341690 T4385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341691 T4385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341691 T4385 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2341692 T4385 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/nodes_used_collection_shard2_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234//nodes_used_collection_shard2_replica1/data/
   [junit4]   2> 2341693 T4385 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2daa38c7
   [junit4]   2> 2341696 T4385 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234//nodes_used_collection_shard2_replica1/data
   [junit4]   2> 2341697 T4385 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234//nodes_used_collection_shard2_replica1/data/index/
   [junit4]   2> 2341698 T4385 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234/nodes_used_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2341699 T4385 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376197615234//nodes_used_collection_shard2_replica1/data/index
   [junit4]   2> 2341705 T4427 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=179991
   [junit4]   2> 2341708 T4445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2341709 T4445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2341710 T4445 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2341711 T4445 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/nodes_used_collection_shard1_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//nodes_used_collection_shard1_replica2/data/
   [junit4]   2> 2341711 T4445 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2daa38c7
   [junit4]   2> 2341713 T4445 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//nodes_used_collection_shard1_replica2/data
   [junit4]   2> 2341714 T4445 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//nodes_used_collection_shard1_replica2/data/index/
   [junit4]   2> 2341714 T4445 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286/nodes_used_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2341715 T4445 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//nodes_used_collection_shard1_replica2/data/index
   [junit4]   2> 2341723 T4385 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1b6a2008 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4e9758d),segFN=segments_1,generation=1}
   [junit4]   2> 2341724 T4385 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2341726 T4445 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1d3b3bef lockFactory=org.apache.lucene.store.NativeFSLockFactory@2169c0b3),segFN=segments_1,generation=1}
   [junit4]   2> 2341727 T4445 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2341736 T4445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2341738 T4385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2341738 T4385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2341744 T4385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2341738 T4445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2341748 T4385 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2341749 T4445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2341749 T4385 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2341750 T4385 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2341746 T4414 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard1/election
   [junit4]   2> 2341750 T4385 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2341750 T4445 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2341752 T4385 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2341753 T4385 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2341752 T4445 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2341755 T4445 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2341758 T4445 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2341759 T4445 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2341760 T4445 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2341772 T4445 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2341774 T4385 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2341792 T4385 oass.SolrIndexSearcher.<init> Opening Searcher@7a426965 main
   [junit4]   2> 2341794 T4385 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2341795 T4385 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2341796 T4445 oass.SolrIndexSearcher.<init> Opening Searcher@3999ae90 main
   [junit4]   2> 2341802 T4459 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica1] Registered new searcher Searcher@7a426965 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2341804 T4445 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2341810 T4445 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2341809 T4385 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica1
   [junit4]   2> 2341816 T4385 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica1 address:http://127.0.0.1:50297/yc_ collection:nodes_used_collection shard:shard2
   [junit4]   2> 2341817 T4385 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 2341819 T4458 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica2] Registered new searcher Searcher@3999ae90 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2341839 T4445 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica2
   [junit4]   2> 2341842 T4445 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica2 address:http://127.0.0.1:50310/yc_ collection:nodes_used_collection shard:shard1
   [junit4]   2> 2341854 T4445 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 2341857 T4414 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2341890 T4414 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=179990
   [junit4]   2> 2342213 T4427 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2342213 T4427 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2342214 T4427 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50307/yc_/nodes_used_collection_shard2_replica2/
   [junit4]   2> 2342214 T4427 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard2_replica2 url=http://127.0.0.1:50307/yc_ START replicas=[http://127.0.0.1:50297/yc_/nodes_used_collection_shard2_replica1/] nUpdates=100
   [junit4]   2> 2342215 T4427 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard2_replica2 url=http://127.0.0.1:50307/yc_ DONE.  We have no versions.  sync failed.
   [junit4]   2> 2342215 T4427 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2342216 T4427 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2342216 T4427 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:50307/yc_/nodes_used_collection_shard2_replica2/ shard2
   [junit4]   2> 2342216 T4427 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leaders/shard2
   [junit4]   2> ASYNC  NEW_CORE C3555 name=nodes_used_collection_shard2_replica1 org.apache.solr.core.SolrCore@5fdf6006 url=http://127.0.0.1:50297/yc_/nodes_used_collection_shard2_replica1 node=127.0.0.1:50297_yc_ C3555_STATE=coll:nodes_used_collection core:nodes_used_collection_shard2_replica1 props:{state=down, core=nodes_used_collection_shard2_replica1, node_name=127.0.0.1:50297_yc_, base_url=http://127.0.0.1:50297/yc_}
   [junit4]   2> 2342249 T4386 C3555 P50297 oasc.SolrCore.execute [nodes_used_collection_shard2_replica1] webapp=/yc_ path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=2 
   [junit4]   2> 2342395 T4414 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2342396 T4414 

[...truncated too long message...]

Test-jetty4-1376197623286//awholynewcollection_0_shard3_replica2/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//awholynewcollection_0_shard3_replica2/data;done=false>>]
   [junit4]   2> 2494519 T4368 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//awholynewcollection_0_shard3_replica2/data
   [junit4]   2> 2494520 T4368 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//awholynewcollection_0_shard3_replica2/data/index [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//awholynewcollection_0_shard3_replica2/data/index;done=false>>]
   [junit4]   2> 2494521 T4368 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376197623286//awholynewcollection_0_shard3_replica2/data/index
   [junit4]   2> 2494528 T4793 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=90190486892511244-127.0.0.1:50310_yc_-n_0000000004) am no longer a leader.
   [junit4]   2> 2494528 T4793 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2494528 T4793 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=90190486892511244-127.0.0.1:50310_yc_-n_0000000004) am no longer a leader.
   [junit4]   2> 2494557 T4368 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/yc_,null}
   [junit4]   2> 2494621 T4368 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2494626 T4368 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50294 50294
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=7E7F9185C22CDD1B -Dtests.slow=true -Dtests.locale=ar_JO -Dtests.timezone=Europe/Oslo -Dtests.file.encoding=UTF-8
   [junit4] ERROR    173s | CollectionsAPIDistributedZkTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_0
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7E7F9185C22CDD1B:FF991F9DB573BD27]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1537)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1551)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:439)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:147)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 2494730 T4368 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 173247 T4367 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> 2495836 T4450 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2495836 T4450 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2495836 T4450 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2495837 T4450 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene42: {}, docValues:{}, sim=DefaultSimilarity, locale=ar_JO, timezone=Europe/Oslo
   [junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=2,threads=2,free=115632528,total=335605760
   [junit4]   2> NOTE: All tests run in this JVM: [BlockCacheTest, TestDocSet, TestFieldTypeCollectionResource, DisMaxRequestHandlerTest, TestIndexSearcher, SolrIndexSplitterTest, DebugComponentTest, TestLFUCache, DeleteShardTest, TestGroupingSearch, PolyFieldTest, TestElisionMultitermQuery, SuggesterTest, IndexSchemaTest, StatsComponentTest, CoreMergeIndexesAdminHandlerTest, TermVectorComponentDistributedTest, TestFuzzyAnalyzedSuggestions, CurrencyFieldOpenExchangeTest, ShardRoutingCustomTest, DistanceFunctionTest, TestSolrXml, ResourceLoaderTest, TestDocumentBuilder, SimplePostToolTest, BasicFunctionalityTest, HardAutoCommitTest, TestWordDelimiterFilterFactory, ZkControllerTest, BasicDistributedZkTest, SliceStateUpdateTest, HdfsChaosMonkeySafeLeaderTest, OverseerCollectionProcessorTest, TestSolrIndexConfig, TestDefaultSearchFieldResource, TestCloudManagedSchema, HdfsBasicDistributedZk2Test, SolrIndexConfigTest, TestCSVResponseWriter, SuggesterTSTTest, TestReplicationHandler, TestSweetSpotSimilarityFactory, FullSolrCloudDistribCmdsTest, TestBM25SimilarityFactory, DirectUpdateHandlerOptimizeTest, ReturnFieldsTest, TestSearchPerf, SpellCheckCollatorTest, NumericFieldsTest, TimeZoneUtilsTest, TestSuggestSpellingConverter, ResponseLogComponentTest, SortByFunctionTest, TestJoin, MBeansHandlerTest, TestPartialUpdateDeduplication, JsonLoaderTest, TestBinaryField, UUIDFieldTest, SolrCmdDistributorTest, SyncSliceTest, TestSystemIdResolver, AnalysisAfterCoreReloadTest, DirectUpdateHandlerTest, LegacyHTMLStripCharFilterTest, TestRangeQuery, FieldAnalysisRequestHandlerTest, TestSchemaNameResource, TestSolrQueryParserDefaultOperatorResource, TestManagedSchema, RAMDirectoryFactoryTest, ShowFileRequestHandlerTest, SchemaVersionSpecificBehaviorTest, LoggingHandlerTest, WordBreakSolrSpellCheckerTest, BadCopyFieldTest, SpellPossibilityIteratorTest, TestCSVLoader, TestSolrJ, TestRealTimeGet, TestDynamicFieldResource, TestPhraseSuggestions, ClusterStateUpdateTest, TestShardHandlerFactory, TestFoldingMultitermQuery, UpdateParamsTest, CoreContainerCoreInitFailuresTest, TestSolrQueryParser, TestFunctionQuery, RecoveryZkTest, TestSolrXmlPersistence, XmlUpdateRequestHandlerTest, AutoCommitTest, NotRequiredUniqueKeyTest, TestQueryUtils, SolrTestCaseJ4Test, TestSurroundQueryParser, SpellCheckComponentTest, TestXIncludeConfig, AddSchemaFieldsUpdateProcessorFactoryTest, FileUtilsTest, CoreAdminHandlerTest, TestRecovery, MultiTermTest, HighlighterTest, TestCloudManagedSchemaAddField, HdfsLockFactoryTest, TestQuerySenderNoQuery, TermsComponentTest, HdfsDirectoryTest, TestCollationField, SolrCoreTest, TestStressReorder, TestValueSourceCache, TestLazyCores, ScriptEngineTest, ConvertedLegacyTest, TestLMJelinekMercerSimilarityFactory, HighlighterConfigTest, EchoParamsTest, TestReload, IndexSchemaRuntimeFieldTest, AssignTest, SliceStateTest, SpellingQueryConverterTest, SpatialFilterTest, TestDefaultSimilarityFactory, BadComponentTest, ParsingFieldUpdateProcessorsTest, TestAnalyzedSuggestions, SearchHandlerTest, TestFieldCollectionResource, PreAnalyzedUpdateProcessorTest, PrimitiveFieldTypeTest, SolrRequestParserTest, RequestHandlersTest, TestMergePolicyConfig, TestSolrDeletionPolicy1, TestFieldTypeResource, TestPluginEnable, CircularListTest, OutputWriterTest, TestPostingsSolrHighlighter, TestRemoteStreaming, TestRandomDVFaceting, UpdateRequestProcessorFactoryTest, TestStressVersions, TestSort, DateMathParserTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed in 174.64s, 1 test, 1 error <<< FAILURES!

[...truncated 500 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:389: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:369: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:449: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1230: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:873: There were test failures: 318 suites, 1345 tests, 1 error, 40 ignored (5 assumptions)

Total time: 119 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure