You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/07/18 18:21:06 UTC

[JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.6.0) - Build # 640 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/640/
Java: 64bit/jdk1.6.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
collection already exists: awholynewcollection_1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_1
	at __randomizedtesting.SeedInfo.seed([200DDDB7A008A888:A1EB53AFD757C8B4]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1522)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:438)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:146)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
	at java.lang.reflect.Method.invoke(Method.java:597)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:680)




Build Log:
[...truncated 9120 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 2186678 T4074 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 2186687 T4074 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1374161822511
   [junit4]   2> 2186690 T4074 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 2186701 T4075 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2186801 T4074 oasc.ZkTestServer.run start zk server on port:53183
   [junit4]   2> 2186814 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2186831 T4081 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3145888 name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2186832 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2186837 T4074 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2186865 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2186872 T4083 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@195110eb name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2186873 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2186873 T4074 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2186883 T4074 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2186893 T4074 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2186902 T4074 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2186922 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2186923 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2186942 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2186944 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2186954 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2186955 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2186965 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2186966 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2186998 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2186999 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2187013 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2187014 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2187037 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2187039 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2187055 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2187057 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2187071 T4074 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2187072 T4074 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2187617 T4074 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2187685 T4074 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53186
   [junit4]   2> 2187686 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2187687 T4074 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2187688 T4074 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914
   [junit4]   2> 2187689 T4074 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/'
   [junit4]   2> 2187746 T4074 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/solr.xml
   [junit4]   2> 2187832 T4074 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2187833 T4074 oasc.CoreContainer.<init> New CoreContainer 691126761
   [junit4]   2> 2187834 T4074 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/]
   [junit4]   2> 2187836 T4074 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2187837 T4074 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2187838 T4074 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2187839 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2187839 T4074 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2187840 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2187840 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2187841 T4074 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2187842 T4074 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2187843 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2187877 T4074 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2187879 T4074 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
   [junit4]   2> 2187879 T4074 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2187883 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2187888 T4094 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1b1bd20 name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2187889 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2187897 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2187936 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2187940 T4096 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@537fcc3c name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2187941 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2187947 T4074 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2187966 T4074 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2187985 T4074 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2187995 T4074 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53186_
   [junit4]   2> 2188000 T4074 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53186_
   [junit4]   2> 2188015 T4074 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2188040 T4074 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2188050 T4074 oasc.Overseer.start Overseer (id=90057069202046979-127.0.0.1:53186_-n_0000000000) starting
   [junit4]   2> 2188063 T4074 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2188083 T4098 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2188090 T4074 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2188100 T4074 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2188113 T4074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2188155 T4097 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2188155 T4074 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914
   [junit4]   2> 2188330 T4074 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/conf/
   [junit4]   2> 2188337 T4074 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2188344 T4074 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2188345 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2188346 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2188399 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 2188416 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2188428 T4101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30eac85f name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2188428 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2188435 T4074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2188447 T4074 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 2188974 T4074 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2188994 T4074 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53190
   [junit4]   2> 2188995 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2188996 T4074 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2188997 T4074 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1374161824269
   [junit4]   2> 2188998 T4074 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1374161824269/'
   [junit4]   2> 2189063 T4074 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1374161824269/solr.xml
   [junit4]   2> 2189147 T4074 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2189154 T4074 oasc.CoreContainer.<init> New CoreContainer 1754244349
   [junit4]   2> 2189155 T4074 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1374161824269/]
   [junit4]   2> 2189157 T4074 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2189158 T4074 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2189159 T4074 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2189159 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2189160 T4074 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2189160 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2189161 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2189161 T4074 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2189162 T4074 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2189163 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2189203 T4074 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2189204 T4074 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
   [junit4]   2> 2189205 T4074 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2189208 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2189213 T4112 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e9913ac name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2189214 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2189220 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2189255 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2189262 T4114 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@60981b0b name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2189263 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2189297 T4074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2190319 T4074 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53190_
   [junit4]   2> 2190324 T4074 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53190_
   [junit4]   2> 2190352 T4101 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2190361 T4114 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2190367 T4096 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 2190378 T4074 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1374161824269
   [junit4]   2> 2190577 T4074 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1374161824269/conf/
   [junit4]   2> 2190587 T4074 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2190589 T4074 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2190591 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2190592 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2191187 T4074 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2191202 T4074 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53193
   [junit4]   2> 2191204 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2191205 T4074 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2191206 T4074 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464
   [junit4]   2> 2191207 T4074 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/'
   [junit4]   2> 2191283 T4074 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/solr.xml
   [junit4]   2> 2191394 T4074 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2191396 T4074 oasc.CoreContainer.<init> New CoreContainer 240622760
   [junit4]   2> 2191397 T4074 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/]
   [junit4]   2> 2191401 T4074 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2191402 T4074 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2191403 T4074 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2191404 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2191405 T4074 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2191406 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2191407 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2191408 T4074 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2191409 T4074 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2191410 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2191471 T4074 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2191472 T4074 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
   [junit4]   2> 2191472 T4074 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2191474 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2191482 T4126 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19ede0f3 name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2191484 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2191499 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2191563 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2191571 T4128 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2cf8d5e0 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2191572 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2191600 T4074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2192620 T4074 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53193_
   [junit4]   2> 2192625 T4074 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53193_
   [junit4]   2> 2192638 T4101 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2192650 T4096 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2192658 T4074 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464
   [junit4]   2> 2192667 T4114 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2192668 T4128 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 2192820 T4074 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/conf/
   [junit4]   2> 2192827 T4074 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2192828 T4074 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2192829 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2192830 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2193395 T4074 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2193401 T4074 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53196
   [junit4]   2> 2193409 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2193411 T4074 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2193412 T4074 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693
   [junit4]   2> 2193412 T4074 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/'
   [junit4]   2> 2193460 T4074 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/solr.xml
   [junit4]   2> 2193581 T4074 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2193582 T4074 oasc.CoreContainer.<init> New CoreContainer 1031858754
   [junit4]   2> 2193583 T4074 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/]
   [junit4]   2> 2193588 T4074 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2193593 T4074 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2193594 T4074 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2193595 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2193596 T4074 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2193597 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2193598 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2193599 T4074 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2193600 T4074 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2193601 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2193676 T4074 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2193677 T4074 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
   [junit4]   2> 2193678 T4074 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2193697 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2193702 T4140 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2351db00 name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2193703 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2193709 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2193749 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2193756 T4142 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e9f6a94 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2193757 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2193777 T4074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2194792 T4074 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53196_
   [junit4]   2> 2194795 T4074 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53196_
   [junit4]   2> 2194816 T4101 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2194817 T4096 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2194831 T4142 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2194833 T4128 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2194834 T4114 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 2194842 T4074 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693
   [junit4]   2> 2194992 T4074 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/conf/
   [junit4]   2> 2194999 T4074 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2195014 T4074 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2195021 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2195022 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2195524 T4074 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2195530 T4074 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53199
   [junit4]   2> 2195532 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 2195533 T4074 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2195534 T4074 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878
   [junit4]   2> 2195535 T4074 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/'
   [junit4]   2> 2195585 T4074 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/solr.xml
   [junit4]   2> 2195661 T4074 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 2195663 T4074 oasc.CoreContainer.<init> New CoreContainer 1617175522
   [junit4]   2> 2195663 T4074 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/]
   [junit4]   2> 2195665 T4074 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 2195666 T4074 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 2195667 T4074 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2195667 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2195668 T4074 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2195668 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2195669 T4074 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2195669 T4074 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2195670 T4074 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2195688 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 2195729 T4074 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2195730 T4074 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53183/solr
   [junit4]   2> 2195731 T4074 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2195733 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2195740 T4154 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@411c321a name:ZooKeeperConnection Watcher:127.0.0.1:53183 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2195743 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2195750 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 2195788 T4074 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2195793 T4156 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71c71eb6 name:ZooKeeperConnection Watcher:127.0.0.1:53183/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2195793 T4074 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2195810 T4074 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2196832 T4074 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53199_
   [junit4]   2> 2196836 T4074 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53199_
   [junit4]   2> 2196865 T4114 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2196865 T4101 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2196865 T4096 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2196866 T4156 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2196886 T4142 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2196886 T4128 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 2196887 T4074 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878
   [junit4]   2> 2197097 T4074 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/conf/
   [junit4]   2> 2197108 T4074 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2197110 T4074 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 2197110 T4074 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2197112 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2197183 T4074 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 2197251 T4106 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&name=nodes_used_collection&replicationFactor=2&action=CREATE&wt=javabin&version=2
   [junit4]   2> 2197258 T4096 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2197265 T4096 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 2197272 T4098 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "maxShardsPerNode":null,
   [junit4]   2> 	  "createNodeSet":null,
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 2197273 T4098 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:53193_
   [junit4]   2> 2197274 T4098 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:53186_
   [junit4]   2> 2197278 T4098 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:53199_
   [junit4]   2> 2197289 T4092 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down
   [junit4]   2> 2197290 T4098 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:53196_
   [junit4]   2> 2197289 T4122 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down
   [junit4]   2> 2197306 T4122 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2197316 T4136 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down
   [junit4]   2> 2197321 T4092 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2197322 T4151 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down
   [junit4]   2> 2197326 T4136 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2197326 T4151 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2198770 T4097 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 2198777 T4097 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2198777 T4097 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53186_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53186"}
   [junit4]   2> 2198778 T4097 oasc.Overseer$ClusterStateUpdater.createCollection Create collection nodes_used_collection with numShards 2
   [junit4]   2> 2198809 T4097 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2198810 T4097 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53193_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53193"}
   [junit4]   2> 2198832 T4097 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2198832 T4097 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53196_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53196"}
   [junit4]   2> 2198857 T4097 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 2198857 T4097 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53199_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53199"}
   [junit4]   2> 2198875 T4114 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2198876 T4096 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2198879 T4128 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2198881 T4142 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2198891 T4101 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2198893 T4156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 2199313 T4122 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 2199315 T4122 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/nodes_used_collection_shard1_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/nodes_used_collection_shard1_replica1/core.properties (No such file or directory)
   [junit4]   2> 2199321 T4122 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/nodes_used_collection_shard1_replica1
   [junit4]   2> 2199326 T4122 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2199325 T4092 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 2199328 T4092 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/nodes_used_collection_shard1_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/nodes_used_collection_shard1_replica2/core.properties (No such file or directory)
   [junit4]   2> 2199329 T4122 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2199329 T4151 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 2199329 T4136 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 2199331 T4151 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/nodes_used_collection_shard2_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/nodes_used_collection_shard2_replica1/core.properties (No such file or directory)
   [junit4]   2> 2199332 T4151 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/nodes_used_collection_shard2_replica1
   [junit4]   2> 2199332 T4151 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2199329 T4092 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/nodes_used_collection_shard1_replica2
   [junit4]   2> 2199333 T4092 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2199332 T4136 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/nodes_used_collection_shard2_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/nodes_used_collection_shard2_replica2/core.properties (No such file or directory)
   [junit4]   2> 2199335 T4136 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/nodes_used_collection_shard2_replica2
   [junit4]   2> 2199335 T4136 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 2199338 T4151 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2199338 T4151 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2199330 T4122 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2199344 T4092 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2199345 T4092 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2199346 T4136 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 2199347 T4136 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 2199355 T4136 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 2199357 T4136 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2199359 T4122 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 2199361 T4122 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2199381 T4151 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 2199381 T4151 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2199383 T4136 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2199400 T4136 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 2199406 T4092 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 2199444 T4151 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2199450 T4092 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2199479 T4122 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 2199496 T4151 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 2199498 T4122 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 2199502 T4092 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 2199717 T4151 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 2199884 T4122 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 2199889 T4136 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 2199962 T4092 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 2200048 T4151 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2200104 T4151 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2200114 T4136 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2200132 T4136 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2200164 T4136 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica2] Schema name=test
   [junit4]   2> 2200198 T4151 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica1] Schema name=test
   [junit4]   2> 2200215 T4122 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2200253 T4092 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2200256 T4122 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2200266 T4092 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 2200283 T4122 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica1] Schema name=test
   [junit4]   2> 2200366 T4092 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica2] Schema name=test
   [junit4]   2> 2203945 T4122 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2203946 T4122 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2203953 T4122 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2203960 T4151 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2203961 T4151 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2203963 T4151 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2203981 T4151 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2204006 T4151 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2204023 T4122 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2204030 T4122 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2204062 T4136 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2204063 T4136 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2204064 T4136 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2204114 T4136 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2204122 T4136 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2204148 T4092 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2204149 T4092 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 2204151 T4092 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2204177 T4092 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2204188 T4092 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2204217 T4151 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204229 T4122 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204243 T4151 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204264 T4122 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204299 T4151 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204318 T4136 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204324 T4122 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204341 T4122 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204344 T4122 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204345 T4122 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204349 T4151 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204350 T4151 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204351 T4151 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204357 T4136 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204368 T4122 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204369 T4122 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204370 T4122 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2204370 T4122 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/nodes_used_collection_shard1_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464//nodes_used_collection_shard1_replica1/data/
   [junit4]   2> 2204371 T4122 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@54d1f0ad
   [junit4]   2> 2204373 T4122 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464//nodes_used_collection_shard1_replica1/data
   [junit4]   2> 2204373 T4122 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464//nodes_used_collection_shard1_replica1/data/index/
   [junit4]   2> 2204374 T4122 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464/nodes_used_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2204375 T4122 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1374161826464//nodes_used_collection_shard1_replica1/data/index
   [junit4]   2> 2204379 T4151 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204384 T4151 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204385 T4151 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2204384 T4136 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204388 T4151 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/nodes_used_collection_shard2_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//nodes_used_collection_shard2_replica1/data/
   [junit4]   2> 2204389 T4151 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@54d1f0ad
   [junit4]   2> 2204390 T4122 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@54709910 lockFactory=org.apache.lucene.store.NativeFSLockFactory@64861c5d),segFN=segments_1,generation=1}
   [junit4]   2> 2204391 T4122 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2204393 T4151 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//nodes_used_collection_shard2_replica1/data
   [junit4]   2> 2204394 T4151 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//nodes_used_collection_shard2_replica1/data/index/
   [junit4]   2> 2204394 T4151 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878/nodes_used_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2204395 T4151 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//nodes_used_collection_shard2_replica1/data/index
   [junit4]   2> 2204397 T4092 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204405 T4136 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204406 T4136 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204406 T4136 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204408 T4122 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2204408 T4151 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1951798d lockFactory=org.apache.lucene.store.NativeFSLockFactory@aa7345b),segFN=segments_1,generation=1}
   [junit4]   2> 2204409 T4151 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2204411 T4122 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2204423 T4122 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2204424 T4122 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2204425 T4136 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204425 T4136 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204426 T4136 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2204427 T4136 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/nodes_used_collection_shard2_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693//nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 2204427 T4136 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@54d1f0ad
   [junit4]   2> 2204431 T4136 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693//nodes_used_collection_shard2_replica2/data
   [junit4]   2> 2204432 T4136 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693//nodes_used_collection_shard2_replica2/data/index/
   [junit4]   2> 2204432 T4136 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693/nodes_used_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2204433 T4136 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1374161828693//nodes_used_collection_shard2_replica2/data/index
   [junit4]   2> 2204425 T4151 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2204437 T4151 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2204439 T4092 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2204429 T4122 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2204440 T4122 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2204441 T4122 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2204442 T4122 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2204439 T4151 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2204445 T4151 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2204444 T4122 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2204447 T4151 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2204448 T4151 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2204448 T4151 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2204450 T4151 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2204451 T4151 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2204460 T4122 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2204460 T4136 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@11664436 lockFactory=org.apache.lucene.store.NativeFSLockFactory@543c4a3),segFN=segments_1,generation=1}
   [junit4]   2> 2204462 T4136 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2204477 T4136 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2204478 T4136 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2204463 T4151 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2204483 T4136 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2204491 T4136 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2204492 T4136 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2204493 T4136 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2204494 T4136 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2204495 T4136 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2204496 T4136 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2204499 T4151 oass.SolrIndexSearcher.<init> Opening Searcher@46904f83 main
   [junit4]   2> 2204501 T4151 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2204502 T4151 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2204503 T4092 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204507 T4163 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica1] Registered new searcher Searcher@46904f83 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2204511 T4136 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2204525 T4122 oass.SolrIndexSearcher.<init> Opening Searcher@2d62dd8c main
   [junit4]   2> 2204534 T4122 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2204535 T4122 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2204536 T4151 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica1
   [junit4]   2> 2204537 T4151 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica1 address:http://127.0.0.1:53199 collection:nodes_used_collection shard:shard2
   [junit4]   2> 2204538 T4151 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
   [junit4]   2> 2204542 T4162 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica1] Registered new searcher Searcher@2d62dd8c main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2204549 T4092 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204551 T4092 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204552 T4092 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2204560 T4122 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica1
   [junit4]   2> 2204564 T4122 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica1 address:http://127.0.0.1:53193 collection:nodes_used_collection shard:shard1
   [junit4]   2> 2204565 T4122 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
   [junit4]   2> 2204566 T4136 oass.SolrIndexSearcher.<init> Opening Searcher@5f37e070 main
   [junit4]   2> 2204568 T4136 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2204569 T4136 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2204574 T4164 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica2] Registered new searcher Searcher@5f37e070 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2204577 T4092 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2204578 T4092 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2204579 T4092 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2204580 T4092 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/nodes_used_collection_shard1_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914//nodes_used_collection_shard1_replica2/data/
   [junit4]   2> 2204580 T4092 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@54d1f0ad
   [junit4]   2> 2204584 T4136 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica2
   [junit4]   2> 2204585 T4136 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica2 address:http://127.0.0.1:53196 collection:nodes_used_collection shard:shard2
   [junit4]   2> 2204586 T4136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
   [junit4]   2> 2204585 T4092 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914//nodes_used_collection_shard1_replica2/data
   [junit4]   2> 2204592 T4092 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914//nodes_used_collection_shard1_replica2/data/index/
   [junit4]   2> 2204592 T4092 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914/nodes_used_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2204593 T4092 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1374161822914//nodes_used_collection_shard1_replica2/data/index
   [junit4]   2> 2204627 T4092 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4edeac9a lockFactory=org.apache.lucene.store.NativeFSLockFactory@5465f4d9),segFN=segments_1,generation=1}
   [junit4]   2> 2204628 T4092 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2204651 T4151 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard2/election
   [junit4]   2> 2204651 T4092 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2204653 T4092 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2204655 T4092 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2204656 T4092 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 2204657 T4092 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 2204658 T4092 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 2204658 T4092 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 2204661 T4092 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 2204662 T4122 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard1/election
   [junit4]   2> 2204662 T4092 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 2204666 T4136 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard2/election
   [junit4]   2> 2204677 T4092 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2204688 T4092 oass.SolrIndexSearcher.<init> Opening Searcher@2ecffeb5 main
   [junit4]   2> 2204690 T4092 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2204690 T4092 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2204709 T4165 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica2] Registered new searcher Searcher@2ecffeb5 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 2204714 T4092 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica2
   [junit4]   2> 2204715 T4092 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica2 address:http://127.0.0.1:53186 collection:nodes_used_collection shard:shard1
   [junit4]   2> 2204723 T4092 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=30000&socketTimeout=30000&retry=false
   [junit4]   2> 2204741 T4151 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 2204769 T4122 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2204776 T4151 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2204782 T4151 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2204783 T4151 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53199/nodes_used_collection_shard2_replica1/
   [junit4]   2> 2204784 T4151 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard2_replica1 url=http://127.0.0.1:53199 START replicas=[http://127.0.0.1:53196/nodes_used_collection_shard2_replica2/] nUpdates=100
   [junit4]   2> 2204785 T4151 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard2_replica1 url=http://127.0.0.1:53199 DONE.  We have no versions.  sync failed.
   [junit4]   2> 2204786 T4151 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2204788 T4151 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2204793 T4151 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53199/nodes_used_collection_shard2_replica1/ shard2
   [junit4]   2> 2204794 T4151 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leaders/shard2
   [junit4]   2> ASYNC  NEW_CORE C2475 name=nodes_used_collection_shard2_replica2 org.apache.solr.core.SolrCore@266d9a4e url=http://127.0.0.1:53196/nodes_used_collection_shard2_replica2 node=127.0.0.1:53196_ C2475_STATE=coll:nodes_used_collection core:nodes_used_collection_shard2_replica2 props:{state=down, core=nodes_used_collection_shard2_replica2, node_name=127.0.0.1:53196_, base_url=http://127.0.0.1:53196}
   [junit4]   2> 2204820 T4134 C2475 P53196 oasc.SolrCore.execute [nodes_used_collection_shard2_replica2] webapp= path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1 
   [junit4]   2> 2204831 T4122 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2204831 T4122 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2204832 T4122 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53193/nodes_used_collection_shard1_replica1/
   [junit4]   2> 2204832 T4122 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard1_replica1 url=http://127.0.0.1:53193 START replicas=[http://127.0.0.1:53186/nodes_used_collection_shard1_replica2/] nUpdates=100
   [junit4]   2> 2204836 T4122 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard1_replica1 url=http://127.0.0.1:53193 DONE.  We have no versions.  sync failed.
   [junit4]   2> 2204836 T4122 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 2204837 T4122 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 2204838 T4122 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53193/nodes_used_collection_shard1_replica1/ shard1
   [junit4]   2> 2204838 T4122 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leaders/shard1
   [junit4]   2> ASYNC  NEW_CORE C2476 name=nodes_used_collection_shard1_replica2 org.apache.solr.core.SolrCore@15288b52 url=http://127.0.0.1:53186/nodes_used_collection_shard1_replica2 node=127.0.0.1:53186_ C2476_STATE=coll:nodes_used_collection core:nodes_used_collection_shard1_replica2 props:{state=down, core=nodes_used_collection_shard1_replica2, node_

[...truncated too long message...]

t4]   2> 2265428 T4074 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2265429 T4074 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//awholynewcollection_1_shard3_replica5/data/index [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//awholynewcollection_1_shard3_replica5/data/index;done=false>>]
   [junit4]   2> 2265429 T4074 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//awholynewcollection_1_shard3_replica5/data/index
   [junit4]   2> 2265430 T4074 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//awholynewcollection_1_shard3_replica5/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//awholynewcollection_1_shard3_replica5/data;done=false>>]
   [junit4]   2> 2265431 T4074 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1374161830878//awholynewcollection_1_shard3_replica5/data
   [junit4]   2> 2265437 T4331 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=90057069202046988-127.0.0.1:53199_-n_0000000004) am no longer a leader.
   [junit4]   2> 2265445 T4156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2265445 T4156 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2265446 T4156 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2265475 T4074 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2265537 T4074 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 2265543 T4074 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:53183 53183
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=200DDDB7A008A888 -Dtests.slow=true -Dtests.locale=en_ZA -Dtests.timezone=Pacific/Rarotonga -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   79.0s | CollectionsAPIDistributedZkTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([200DDDB7A008A888:A1EB53AFD757C8B4]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1522)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:438)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:146)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:680)
   [junit4]   2> 2265720 T4074 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 79060 T4073 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene42: {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=en_ZA, timezone=Pacific/Rarotonga
   [junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Apple Inc. 1.6.0_51 (64-bit)/cpus=2,threads=2,free=132977568,total=267132928
   [junit4]   2> NOTE: All tests run in this JVM: [UniqFieldsUpdateProcessorFactoryTest, TestSchemaSimilarityResource, TestDocumentBuilder, DirectSolrSpellCheckerTest, DOMUtilTest, NoCacheHeaderTest, TestSolrXmlPersistence, TestShardHandlerFactory, TestSolrDeletionPolicy2, NumericFieldsTest, TestDistributedSearch, HdfsCollectionsAPIDistributedZkTest, TestCloudManagedSchemaAddField, TestOmitPositions, TestFiltering, TestNumberUtils, TestRemoteStreaming, TestBM25SimilarityFactory, TestDocSet, FastVectorHighlighterTest, TestSolrQueryParserDefaultOperatorResource, LoggingHandlerTest, SortByFunctionTest, TestSolrJ, TestSerializedLuceneMatchVersion, TestCollationKeyRangeQueries, TestLMDirichletSimilarityFactory, ReturnFieldsTest, ScriptEngineTest, TestInfoStreamLogging, JsonLoaderTest, RequestHandlersTest, TestLFUCache, SuggesterWFSTTest, TestReloadAndDeleteDocs, TestReversedWildcardFilterFactory, CSVRequestHandlerTest, TestSolrXMLSerializer, TestPropInject, FieldAnalysisRequestHandlerTest, SimplePostToolTest, BasicZkTest, HdfsLockFactoryTest, LukeRequestHandlerTest, AlternateDirectoryTest, TestWriterPerf, TestPerFieldSimilarity, TestPseudoReturnFields, OutputWriterTest, TestSweetSpotSimilarityFactory, ZkControllerTest, SpellPossibilityIteratorTest, TestIndexSearcher, FullSolrCloudDistribCmdsTest, TestQuerySenderNoQuery, TestCoreContainer, RequiredFieldsTest, PrimitiveFieldTypeTest, TestComponentsName, AddSchemaFieldsUpdateProcessorFactoryTest, AnalysisAfterCoreReloadTest, PreAnalyzedUpdateProcessorTest, DistributedSpellCheckComponentTest, TestMultiCoreConfBootstrap, ShowFileRequestHandlerTest, BasicDistributedZkTest, ClusterStateUpdateTest, CurrencyFieldOpenExchangeTest, UUIDFieldTest, TestValueSourceCache, SuggesterFSTTest, ZkSolrClientTest, PrimUtilsTest, TestSystemIdResolver, PolyFieldTest, IndexBasedSpellCheckerTest, ResourceLoaderTest, TestDynamicFieldCollectionResource, TestDefaultSearchFieldResource, HdfsChaosMonkeySafeLeaderTest, TestPostingsSolrHighlighter, TestRecoveryHdfs, SolrTestCaseJ4Test, SampleTest, CoreAdminHandlerTest, DateFieldTest, NotRequiredUniqueKeyTest, TimeZoneUtilsTest, HdfsBasicDistributedZk2Test, DocumentAnalysisRequestHandlerTest, TestCopyFieldCollectionResource, DebugComponentTest, SliceStateUpdateTest, TestArbitraryIndexDir, TestAnalyzedSuggestions, SuggesterTest, TestRecovery, TestSuggestSpellingConverter, TestZkChroot, TestJoin, TestJmxMonitoredMap, HdfsUnloadDistributedZkTest, LegacyHTMLStripCharFilterTest, DateMathParserTest, TestBinaryResponseWriter, TestGroupingSearch, QueryElevationComponentTest, TestSolrQueryParser, QueryParsingTest, HdfsSyncSliceTest, BadComponentTest, ZkCLITest, HdfsDirectoryTest, CacheHeaderTest, ShardRoutingTest, ResponseLogComponentTest, TestDefaultSimilarityFactory, PreAnalyzedFieldTest, SoftAutoCommitTest, HdfsRecoveryZkTest, SpellCheckComponentTest, DocumentBuilderTest, SchemaVersionSpecificBehaviorTest, SpellingQueryConverterTest, TestFieldTypeCollectionResource, TermVectorComponentTest, PluginInfoTest, FieldMutatingUpdateProcessorTest, DocValuesMultiTest, SolrCmdDistributorTest, SpatialFilterTest, TestPhraseSuggestions, TestSolrIndexConfig, TestJmxIntegration, LeaderElectionTest, DistributedQueryElevationComponentTest, CircularListTest, MoreLikeThisHandlerTest, ExternalFileFieldSortTest, BlockCacheTest, EchoParamsTest, TestQueryUtils, DistanceFunctionTest, StressHdfsTest, TestManagedSchema, PingRequestHandlerTest, IndexReaderFactoryTest, TestDistributedGrouping, TestManagedSchemaFieldResource, QueryEqualityTest, IndexSchemaRuntimeFieldTest, FileUtilsTest, MultiTermTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed in 79.66s, 1 test, 1 error <<< FAILURES!

[...truncated 500 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/build.xml:395: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/build.xml:375: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:449: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1250: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:893: There were test failures: 319 suites, 1345 tests, 1 error, 39 ignored (5 assumptions)

Total time: 128 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.6.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure