You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/14 01:20:41 UTC

[JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 709 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/709/
Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
Server at http://127.0.0.1:52362/y_/c returned non ok status:500, message:Server Error

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:52362/y_/c returned non ok status:500, message:Server Error
	at __randomizedtesting.SeedInfo.seed([7F7164A0CA384F3D:FE97EAB8BD672F01]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection(CollectionsAPIDistributedZkTest.java:185)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:149)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 9744 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 826546 T1241 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /y_/c
   [junit4]   2> 826561 T1241 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1376432958363
   [junit4]   2> 826563 T1241 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 826564 T1242 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 826665 T1241 oasc.ZkTestServer.run start zk server on port:52355
   [junit4]   2> 826672 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 826680 T1248 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e2e7e79 name:ZooKeeperConnection Watcher:127.0.0.1:52355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 826680 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 826681 T1241 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 826695 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 826705 T1250 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f7704ff name:ZooKeeperConnection Watcher:127.0.0.1:52355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 826705 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 826706 T1241 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 826716 T1241 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 826804 T1241 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 826811 T1241 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 826826 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 826827 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 826838 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 826838 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 826846 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 826847 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 826854 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 826854 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 826863 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 826864 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 826872 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 826873 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 826879 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 826880 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 826887 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 826891 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 826898 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 826898 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 826908 T1241 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 826909 T1241 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 826927 T1243 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14079cc2ba80001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 
   [junit4]   2> 827375 T1241 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 827384 T1241 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:52358
   [junit4]   2> 827385 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 827386 T1241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 827386 T1241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719
   [junit4]   2> 827387 T1241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/'
   [junit4]   2> 827435 T1241 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/solr.xml
   [junit4]   2> 827557 T1241 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 827558 T1241 oasc.CoreContainer.<init> New CoreContainer 396183743
   [junit4]   2> 827558 T1241 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/]
   [junit4]   2> 827560 T1241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 827561 T1241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 827561 T1241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 827562 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 827562 T1241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 827562 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 827563 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 827563 T1241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 827564 T1241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 827564 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 827587 T1241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52355/solr
   [junit4]   2> 827587 T1241 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 827589 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 827595 T1261 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6393d737 name:ZooKeeperConnection Watcher:127.0.0.1:52355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 827596 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 827602 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 827627 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 827631 T1263 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21b56626 name:ZooKeeperConnection Watcher:127.0.0.1:52355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 827632 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 827636 T1241 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 827645 T1241 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 827653 T1241 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 827659 T1241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52358_y_%2Fc
   [junit4]   2> 827662 T1241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52358_y_%2Fc
   [junit4]   2> 827672 T1241 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 827685 T1241 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 827697 T1241 oasc.Overseer.start Overseer (id=90205910360129539-127.0.0.1:52358_y_%2Fc-n_0000000000) starting
   [junit4]   2> 827705 T1241 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 827715 T1265 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 827720 T1241 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 827730 T1241 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 827735 T1241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 827752 T1241 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719
   [junit4]   2> 827758 T1264 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 827914 T1241 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/conf/
   [junit4]   2> 827930 T1241 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 827932 T1241 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 827932 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 827932 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 827954 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 827957 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 827969 T1268 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3eebc55 name:ZooKeeperConnection Watcher:127.0.0.1:52355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 827970 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 827974 T1241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 827980 T1241 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 828428 T1241 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 828436 T1241 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:52362
   [junit4]   2> 828437 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 828438 T1241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 828438 T1241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376432959781
   [junit4]   2> 828438 T1241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376432959781/'
   [junit4]   2> 828479 T1241 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376432959781/solr.xml
   [junit4]   2> 828615 T1241 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 828616 T1241 oasc.CoreContainer.<init> New CoreContainer 1140603759
   [junit4]   2> 828617 T1241 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376432959781/]
   [junit4]   2> 828618 T1241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 828619 T1241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 828619 T1241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 828620 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 828620 T1241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 828621 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 828623 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 828625 T1241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 828627 T1241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 828628 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 828649 T1241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52355/solr
   [junit4]   2> 828650 T1241 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 828652 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 828657 T1279 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6874b015 name:ZooKeeperConnection Watcher:127.0.0.1:52355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 828657 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 828664 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 828682 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 828687 T1281 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@549f5b38 name:ZooKeeperConnection Watcher:127.0.0.1:52355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 828687 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 828706 T1241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 829715 T1241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52362_y_%2Fc
   [junit4]   2> 829725 T1241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52362_y_%2Fc
   [junit4]   2> 829738 T1281 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 829742 T1263 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 829743 T1268 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 829750 T1241 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376432959781
   [junit4]   2> 829901 T1241 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1376432959781/conf/
   [junit4]   2> 829907 T1241 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 829908 T1241 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 829909 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 829909 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 830388 T1241 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 830397 T1241 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:52365
   [junit4]   2> 830397 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 830398 T1241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 830398 T1241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730
   [junit4]   2> 830399 T1241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/'
   [junit4]   2> 830450 T1241 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/solr.xml
   [junit4]   2> 830580 T1241 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 830580 T1241 oasc.CoreContainer.<init> New CoreContainer 2059622291
   [junit4]   2> 830581 T1241 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/]
   [junit4]   2> 830583 T1241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 830583 T1241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 830584 T1241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 830584 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 830585 T1241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 830585 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 830586 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 830586 T1241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 830587 T1241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 830588 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 830607 T1241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52355/solr
   [junit4]   2> 830608 T1241 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 830613 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 830616 T1293 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6543aa57 name:ZooKeeperConnection Watcher:127.0.0.1:52355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 830616 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 830628 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 830644 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 830648 T1295 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ebbf07a name:ZooKeeperConnection Watcher:127.0.0.1:52355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 830648 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 830661 T1241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 831671 T1241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52365_y_%2Fc
   [junit4]   2> 831673 T1241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52365_y_%2Fc
   [junit4]   2> 831683 T1263 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 831683 T1295 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 831690 T1268 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 831691 T1281 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 831701 T1241 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730
   [junit4]   2> 831863 T1241 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/conf/
   [junit4]   2> 831870 T1241 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 831871 T1241 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 831871 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 831876 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 832424 T1241 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 832436 T1241 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:52368
   [junit4]   2> 832437 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 832437 T1241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 832438 T1241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706
   [junit4]   2> 832438 T1241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/'
   [junit4]   2> 832493 T1241 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/solr.xml
   [junit4]   2> 832616 T1241 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 832622 T1241 oasc.CoreContainer.<init> New CoreContainer 652334541
   [junit4]   2> 832623 T1241 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/]
   [junit4]   2> 832625 T1241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 832625 T1241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 832626 T1241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 832626 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 832626 T1241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 832627 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 832627 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 832628 T1241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 832628 T1241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 832629 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 832648 T1241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52355/solr
   [junit4]   2> 832648 T1241 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 832652 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 832655 T1307 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4002280 name:ZooKeeperConnection Watcher:127.0.0.1:52355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 832656 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 832663 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 832682 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 832689 T1309 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e52e33f name:ZooKeeperConnection Watcher:127.0.0.1:52355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 832690 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 832703 T1241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 833711 T1241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52368_y_%2Fc
   [junit4]   2> 833715 T1241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52368_y_%2Fc
   [junit4]   2> 833724 T1268 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 833724 T1263 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 833731 T1295 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 833732 T1281 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 833735 T1309 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 833751 T1241 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706
   [junit4]   2> 833925 T1241 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/conf/
   [junit4]   2> 833932 T1241 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 833933 T1241 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 833933 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 833934 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 834417 T1241 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 834435 T1241 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:52371
   [junit4]   2> 834436 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 834436 T1241 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 834437 T1241 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756
   [junit4]   2> 834437 T1241 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/'
   [junit4]   2> 834481 T1241 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/solr.xml
   [junit4]   2> 834588 T1241 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 834589 T1241 oasc.CoreContainer.<init> New CoreContainer 1736183144
   [junit4]   2> 834590 T1241 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/]
   [junit4]   2> 834640 T1241 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 834640 T1241 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 834641 T1241 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 834641 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 834641 T1241 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 834644 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 834645 T1241 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 834645 T1241 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 834646 T1241 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 834647 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 834671 T1241 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52355/solr
   [junit4]   2> 834671 T1241 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 834675 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 834678 T1321 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f9a864 name:ZooKeeperConnection Watcher:127.0.0.1:52355 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 834679 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 834684 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 834710 T1241 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 834714 T1323 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ed57c3f name:ZooKeeperConnection Watcher:127.0.0.1:52355/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 834714 T1241 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 834726 T1241 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 835736 T1241 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52371_y_%2Fc
   [junit4]   2> 835738 T1241 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52371_y_%2Fc
   [junit4]   2> 835747 T1263 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 835747 T1268 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 835755 T1295 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 835756 T1309 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 835757 T1281 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 835765 T1241 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756
   [junit4]   2> 835768 T1323 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 835917 T1241 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/conf/
   [junit4]   2> 835924 T1241 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 835925 T1241 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 835925 T1241 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 835926 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 835946 T1241 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 835970 T1277 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&name=nodes_used_collection&replicationFactor=2&action=CREATE&wt=javabin&version=2
   [junit4]   2> 835973 T1263 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 835976 T1265 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 835977 T1265 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 836803 T1264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 836805 T1264 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: nodes_used_collection
   [junit4]   2> 836805 T1264 oasc.Overseer$ClusterStateUpdater.createCollection Create collection nodes_used_collection with shards [shard1, shard2]
   [junit4]   2> 836819 T1263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 836819 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 836819 T1281 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 836820 T1268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 836820 T1323 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 836821 T1309 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 836894 T1265 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 836894 T1265 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:52368_y_%2Fc
   [junit4]   2> 836895 T1265 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:52358_y_%2Fc
   [junit4]   2> 836896 T1265 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:52371_y_%2Fc
   [junit4]   2> 836897 T1265 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:52365_y_%2Fc
   [junit4]   2> 836908 T1291 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down
   [junit4]   2> 836919 T1291 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 836922 T1305 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down
   [junit4]   2> 836922 T1259 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down
   [junit4]   2> 836924 T1315 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down
   [junit4]   2> 836926 T1305 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 836927 T1315 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 836927 T1259 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 838339 T1264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 838342 T1264 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 838342 T1264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52365_y_%2Fc",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52365/y_/c"}
   [junit4]   2> 838351 T1264 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 838352 T1264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52358_y_%2Fc",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52358/y_/c"}
   [junit4]   2> 838368 T1264 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 838369 T1264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52368_y_%2Fc",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52368/y_/c"}
   [junit4]   2> 838378 T1264 oasc.Overseer$ClusterStateUpdater.updateState WARN Could not find collection node for nodes_used_collection, skipping publish state
   [junit4]   2> 838378 T1264 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52371_y_%2Fc",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52371/y_/c"}
   [junit4]   2> 838389 T1309 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 838389 T1323 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 838392 T1281 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 838392 T1295 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 838392 T1268 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 838393 T1263 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 838924 T1291 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 838925 T1291 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/nodes_used_collection_shard2_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/nodes_used_collection_shard2_replica2/core.properties (No such file or directory)
   [junit4]   2> 838925 T1291 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/nodes_used_collection_shard2_replica2
   [junit4]   2> 838926 T1291 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 838928 T1291 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 838928 T1291 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 838929 T1315 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 838930 T1315 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/nodes_used_collection_shard2_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/nodes_used_collection_shard2_replica1/core.properties (No such file or directory)
   [junit4]   2> 838930 T1315 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/nodes_used_collection_shard2_replica1
   [junit4]   2> 838931 T1315 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 838931 T1259 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 838932 T1259 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/nodes_used_collection_shard1_replica2/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/nodes_used_collection_shard1_replica2/core.properties (No such file or directory)
   [junit4]   2> 838932 T1259 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica2' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/nodes_used_collection_shard1_replica2
   [junit4]   2> 838933 T1259 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 838936 T1259 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 838936 T1259 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 838938 T1291 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 838938 T1291 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 838947 T1305 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 838950 T1315 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 838950 T1315 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 838951 T1259 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 838951 T1259 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 838953 T1305 oasc.CorePropertiesLocator.writePropertiesFile ERROR Couldn't persist core properties to /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/nodes_used_collection_shard1_replica1/core.properties: java.io.FileNotFoundException: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/nodes_used_collection_shard1_replica1/core.properties (No such file or directory)
   [junit4]   2> 838953 T1305 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using instanceDir: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/nodes_used_collection_shard1_replica1
   [junit4]   2> 838954 T1305 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 838958 T1305 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 838959 T1305 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 838962 T1305 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 838985 T1291 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 838992 T1315 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 838996 T1259 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 838997 T1291 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 839004 T1315 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 839014 T1259 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 839015 T1315 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 839258 T1291 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 839285 T1315 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 839347 T1259 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 839355 T1305 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 839555 T1291 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 839563 T1291 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 839564 T1315 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 839573 T1315 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 839603 T1305 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 839611 T1291 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica2] Schema name=test
   [junit4]   2> 839619 T1305 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 839619 T1315 oass.IndexSchema.readSchema [nodes_used_collection_shard2_replica1] Schema name=test
   [junit4]   2> 839640 T1259 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 839643 T1259 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 839666 T1305 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica1] Schema name=test
   [junit4]   2> 839718 T1259 oass.IndexSchema.readSchema [nodes_used_collection_shard1_replica2] Schema name=test
   [junit4]   2> 842407 T1305 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842408 T1305 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842409 T1305 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 842472 T1291 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842480 T1291 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842481 T1291 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 842553 T1305 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 842558 T1305 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 842568 T1291 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 842579 T1259 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842580 T1259 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842581 T1259 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 842588 T1291 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 842633 T1305 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842641 T1259 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 842642 T1305 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842648 T1259 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 842650 T1305 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842654 T1305 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842654 T1305 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842654 T1305 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842658 T1305 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842659 T1305 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842659 T1305 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 842660 T1305 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/nodes_used_collection_shard1_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706//nodes_used_collection_shard1_replica1/data/
   [junit4]   2> 842660 T1305 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@e7e77c9
   [junit4]   2> 842660 T1291 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842683 T1305 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706//nodes_used_collection_shard1_replica1/data
   [junit4]   2> 842684 T1305 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706//nodes_used_collection_shard1_replica1/data/index/
   [junit4]   2> 842685 T1305 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706/nodes_used_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 842689 T1315 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842689 T1315 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 842690 T1315 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 842690 T1291 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842698 T1305 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1376432963706//nodes_used_collection_shard1_replica1/data/index
   [junit4]   2> 842706 T1291 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842710 T1315 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 842716 T1305 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4c1c68d3 lockFactory=org.apache.lucene.store.NativeFSLockFactory@62da24de),segFN=segments_1,generation=1}
   [junit4]   2> 842720 T1305 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 842718 T1315 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 842716 T1291 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842729 T1291 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842730 T1291 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842745 T1291 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842746 T1291 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842746 T1291 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 842747 T1291 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/nodes_used_collection_shard2_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730//nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 842747 T1291 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@e7e77c9
   [junit4]   2> 842755 T1305 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 842756 T1305 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 842758 T1305 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 842759 T1305 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 842761 T1305 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 842761 T1305 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 842761 T1305 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 842762 T1305 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 842763 T1305 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 842759 T1291 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730//nodes_used_collection_shard2_replica2/data
   [junit4]   2> 842768 T1291 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730//nodes_used_collection_shard2_replica2/data/index/
   [junit4]   2> 842768 T1291 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730/nodes_used_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 842770 T1259 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842776 T1291 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1376432961730//nodes_used_collection_shard2_replica2/data/index
   [junit4]   2> 842781 T1291 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5556f1e7 lockFactory=org.apache.lucene.store.NativeFSLockFactory@38054953),segFN=segments_1,generation=1}
   [junit4]   2> 842781 T1291 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 842791 T1291 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 842791 T1291 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 842792 T1291 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 842793 T1291 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 842794 T1291 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 842795 T1291 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 842795 T1291 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 842796 T1291 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 842797 T1291 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 842804 T1305 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 842807 T1259 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842817 T1259 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842824 T1305 oass.SolrIndexSearcher.<init> Opening Searcher@5d9e36b7 main
   [junit4]   2> 842826 T1305 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 842826 T1305 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 842827 T1259 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842827 T1259 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842840 T1259 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842829 T1291 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 842844 T1329 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica1] Registered new searcher Searcher@5d9e36b7 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 842848 T1291 oass.SolrIndexSearcher.<init> Opening Searcher@46a4e1e8 main
   [junit4]   2> 842848 T1305 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica1
   [junit4]   2> 842848 T1305 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica1 address:http://127.0.0.1:52368/y_/c collection:nodes_used_collection shard:shard1
   [junit4]   2> 842849 T1305 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 842861 T1259 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842862 T1259 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842862 T1259 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 842863 T1259 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica2] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/nodes_used_collection_shard1_replica2/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719//nodes_used_collection_shard1_replica2/data/
   [junit4]   2> 842863 T1259 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@e7e77c9
   [junit4]   2> 842849 T1291 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 842865 T1291 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 842875 T1259 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719//nodes_used_collection_shard1_replica2/data
   [junit4]   2> 842875 T1259 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719//nodes_used_collection_shard1_replica2/data/index/
   [junit4]   2> 842876 T1259 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica2] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719/nodes_used_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 842878 T1330 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica2] Registered new searcher Searcher@46a4e1e8 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 842881 T1291 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica2
   [junit4]   2> 842881 T1291 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica2 address:http://127.0.0.1:52365/y_/c collection:nodes_used_collection shard:shard2
   [junit4]   2> 842881 T1291 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 842888 T1259 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1376432958719//nodes_used_collection_shard1_replica2/data/index
   [junit4]   2> 842904 T1315 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842909 T1291 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard2/election
   [junit4]   2> 842911 T1259 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@5b222258 lockFactory=org.apache.lucene.store.NativeFSLockFactory@c69c38),segFN=segments_1,generation=1}
   [junit4]   2> 842911 T1259 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 842925 T1259 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 842926 T1259 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 842927 T1259 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 842928 T1259 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 842929 T1259 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 842929 T1259 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 842929 T1259 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 842930 T1259 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 842931 T1259 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 842933 T1315 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 842936 T1305 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard1/election
   [junit4]   2> 842953 T1315 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842960 T1259 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 842966 T1315 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842966 T1315 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842967 T1315 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 842968 T1259 oass.SolrIndexSearcher.<init> Opening Searcher@455ba907 main
   [junit4]   2> 842970 T1259 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 842976 T1259 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 842977 T1315 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 842979 T1315 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 842979 T1315 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 842980 T1315 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica1] Opening new SolrCore at /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/nodes_used_collection_shard2_replica1/, dataDir=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//nodes_used_collection_shard2_replica1/data/
   [junit4]   2> 842980 T1315 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@e7e77c9
   [junit4]   2> 842982 T1331 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica2] Registered new searcher Searcher@455ba907 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 843015 T1259 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica2
   [junit4]   2> 843016 T1259 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica2 address:http://127.0.0.1:52358/y_/c collection:nodes_used_collection shard:shard1
   [junit4]   2> 843016 T1259 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 843020 T1315 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//nodes_used_collection_shard2_replica1/data
   [junit4]   2> 843025 T1315 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//nodes_used_collection_shard2_replica1/data/index/
   [junit4]   2> 843025 T1315 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica1] Solr index directory '/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756/nodes_used_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 843038 T1315 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//nodes_used_collection_shard2_replica1/data/index
   [junit4]   2> 843052 T1315 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@53c371f1 lockFactory=org.apache.lucene.store.NativeFSLockFactory@63cb4947),segFN=segments_1,generation=1}
   [junit4]   2> 843052 T1315 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 843061 T1315 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 843062 T1315 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 843063 T1315 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 843065 T1315 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 843066 T1305 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 843069 T1315 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 843070 T1315 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 843071 T1291 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 843080 T1315 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 843081 T1315 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 843081 T1315 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 843085 T1305 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 843085 T1305 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 843086 T1305 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52368/y_/c/nodes_used_collection_shard1_replica1/
   [junit4]   2> 843087 T1305 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard1_replica1 url=http://127.0.0.1:52368/y_/c START replicas=[http://127.0.0.1:52358/y_/c/nodes_used_collection_shard1_replica2/] nUpdates=100
   [junit4]   2> 843089 T1305 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard1_replica1 url=http://127.0.0.1:52368/y_/c DONE.  We have no versions.  sync failed.
   [junit4]   2> ASYNC  NEW_CORE C843 name=nodes_used_collection_shard1_replica2 org.apache.solr.core.SolrCore@72eb6a93 url=http://127.0.0.1:52358/y_/c/nodes_used_collection_shard1_replica2 node=127.0.0.1:52358_y_%2Fc C843_STATE=coll:nodes_used_collection core:nodes_used_collection_shard1_replica2 props:{state=down, core=nodes_used_collection_shard1_replica2, node_name=127.0.0.1:52358_y_%2Fc, base_url=http://127.0.0.1:52358/y_/c}
   [junit4]   2> 843104 T1254 C843 P52358 oasc.SolrCore.execute [nodes_used_collection_shard1_replica2] webapp=/y_/c path=/get params={getVersions=100&distrib=false&wt=javabin&qt=/get&version=2} status=0 QTime=1 
   [junit4]   2> 843105 T1305 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 843106 T1305 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 843106 T1305 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:52368/y_/c/nodes_used_collection_shard1_replica1/ shard1
   [junit4]   2> 843107 T1305 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leaders/shard1
   [junit4]   2> 843114 T1291 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=179994
   [junit4]   2> 843126 T1315 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 843130 T1315 oass.SolrIndexSearcher.<init> Opening Searcher@1ac6a735 main
   [junit4]   2> 843132 T1315 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 843132 T1315 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 843146 T1333 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica1] Registered new searcher Searcher@1ac6a735 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 843150 T1315 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica1
   [junit4]   2> 843151 T1315 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica1 address:http://127.0.0.1:52371/y_/c collection:nodes_used_collection shard:shard2
   [junit4]   2> 843151 T1315 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 843620 T1291 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 843622 T1291 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 843623 T1291 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52365/y_/c/nodes_used_collection_shard2_replica2/
   [junit4]   2> 843624 T1291 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard2_replica2 url=http://127.0.0.1:52365/y_/c START replicas=[http://127.0.0.1:52371/y_/c/nodes_used_collection_shard2_replica1/] nUpdates=100
   [junit4]   

[...truncated too long message...]

y.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard2_replica1/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard2_replica1/data;done=false>>]
   [junit4]   2> 1160408 T1241 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard2_replica1/data
   [junit4]   2> 1160409 T1241 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard2_replica1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard2_replica1/data/index;done=false>>]
   [junit4]   2> 1160409 T1241 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard2_replica1/data/index
   [junit4]   2> 1160410 T1241 oasc.SolrCore.close [awholynewcollection_4_shard4_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3429abe4
   [junit4]   2> 1160427 T1241 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1160427 T1241 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1160428 T1241 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1160428 T1241 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1160429 T1241 oasc.SolrCore.closeSearcher [awholynewcollection_4_shard4_replica1] Closing main searcher on request.
   [junit4]   2> 1160430 T1241 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1160431 T1241 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard4_replica1/data/index [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard4_replica1/data/index;done=false>>]
   [junit4]   2> 1160431 T1241 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard4_replica1/data/index
   [junit4]   2> 1160431 T1241 oasc.CachingDirectoryFactory.closeCacheValue looking to close /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard4_replica1/data [CachedDir<<refCount=0;path=/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard4_replica1/data;done=false>>]
   [junit4]   2> 1160432 T1241 oasc.CachingDirectoryFactory.close Closing directory: /Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1376432965756//awholynewcollection_4_shard4_replica1/data
   [junit4]   2> 1160437 T1570 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=90205910360129548-127.0.0.1:52371_y_%2Fc-n_0000000004) am no longer a leader.
   [junit4]   2> 1160467 T1241 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/y_/c,null}
   [junit4]   2> 1160535 T1241 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1160538 T1241 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:52355 52355
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=7F7164A0CA384F3D -Dtests.slow=true -Dtests.locale=fr_LU -Dtests.timezone=Europe/Vatican -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    334s | CollectionsAPIDistributedZkTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:52362/y_/c returned non ok status:500, message:Server Error
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7F7164A0CA384F3D:FE97EAB8BD672F01]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.deletePartiallyCreatedCollection(CollectionsAPIDistributedZkTest.java:185)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:149)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 1160659 T1241 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 334128 T1240 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> 1161742 T1323 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1161742 T1323 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1161743 T1323 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene42: {timestamp=Lucene41(blocksize=128), range_facet_l=Lucene41(blocksize=128), id=PostingsFormat(name=SimpleText), range_facet_sl=PostingsFormat(name=SimpleText), a_t=Lucene41(blocksize=128), text=PostingsFormat(name=MockRandom), range_facet_si=Lucene41(blocksize=128), _version_=PostingsFormat(name=TestBloomFilteredLucene41Postings), other_tl1=PostingsFormat(name=TestBloomFilteredLucene41Postings), multiDefault=Lucene41(blocksize=128), a_si=PostingsFormat(name=TestBloomFilteredLucene41Postings), intDefault=PostingsFormat(name=TestBloomFilteredLucene41Postings)}, docValues:{}, sim=DefaultSimilarity, locale=fr_LU, timezone=Europe/Vatican
   [junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=2,threads=3,free=46361392,total=157884416
   [junit4]   2> NOTE: All tests run in this JVM: [ZkNodePropsTest, ShardSplitTest, NoCacheHeaderTest, HdfsLockFactoryTest, PingRequestHandlerTest, DocumentBuilderTest, HdfsRecoveryZkTest, TestPartialUpdateDeduplication, CurrencyFieldXmlFileTest, TestRecovery, ConvertedLegacyTest, CacheHeaderTest, TestDistributedSearch, TestCSVResponseWriter, UnloadDistributedZkTest, DirectUpdateHandlerTest, PluginInfoTest, BasicDistributedZkTest, DOMUtilTest, HardAutoCommitTest, TestSolrIndexConfig, TestStressLucene, XsltUpdateRequestHandlerTest, TestRangeQuery, TestFieldResource, LoggingHandlerTest, TestRandomMergePolicy, TestJmxMonitoredMap, TestReloadAndDeleteDocs, ShowFileRequestHandlerTest, TestQuerySenderListener, TermVectorComponentDistributedTest, SolrTestCaseJ4Test, TestSolrDeletionPolicy2, CollectionsAPIDistributedZkTest]
   [junit4] Completed in 335.45s, 1 test, 1 error <<< FAILURES!

[...truncated 886 lines...]
BUILD FAILED
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/build.xml:395: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/build.xml:375: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:450: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1233: The following error occurred while executing this line:
/Users/jenkins/jenkins-slave/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:876: There were test failures: 321 suites, 1363 tests, 1 error, 41 ignored (6 assumptions)

Total time: 126 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure