You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/28 23:38:58 UTC

[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 776 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/776/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
collection already exists: awholynewcollection_1

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_1
	at __randomizedtesting.SeedInfo.seed([19679AFCD04CEFCA:988114E4A7138FF6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1571)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1585)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:481)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:152)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 9898 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 1899740 T4436 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_kiw/qj
   [junit4]   2> 1899751 T4436 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1377724230879
   [junit4]   2> 1899753 T4436 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1899754 T4437 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1899857 T4436 oasc.ZkTestServer.run start zk server on port:50394
   [junit4]   2> 1899865 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1899877 T4443 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21c70ebd name:ZooKeeperConnection Watcher:127.0.0.1:50394 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1899877 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1899879 T4436 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1899895 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1899902 T4445 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@483da44a name:ZooKeeperConnection Watcher:127.0.0.1:50394/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1899903 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1899903 T4436 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1899914 T4436 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1899923 T4436 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1899930 T4436 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1899941 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1899942 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1899959 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1899960 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1899970 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1899972 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1899982 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1899983 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1899991 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1899993 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1900001 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1900002 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1900011 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1900012 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1900020 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1900021 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1900031 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1900032 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1900040 T4436 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1900042 T4436 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1900478 T4436 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1900533 T4436 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50397
   [junit4]   2> 1900540 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1900541 T4436 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1900546 T4436 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179
   [junit4]   2> 1900546 T4436 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/'
   [junit4]   2> 1900617 T4436 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/solr.xml
   [junit4]   2> 1900777 T4436 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1900778 T4436 oasc.CoreContainer.<init> New CoreContainer 1219253634
   [junit4]   2> 1900779 T4436 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/]
   [junit4]   2> 1900781 T4436 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1900783 T4436 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1900783 T4436 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1900784 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1900784 T4436 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1900785 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1900785 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1900786 T4436 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1900786 T4436 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1900787 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1900809 T4436 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1900810 T4436 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1900811 T4436 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50394/solr
   [junit4]   2> 1900812 T4436 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1900814 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1900819 T4456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a5f5bd8 name:ZooKeeperConnection Watcher:127.0.0.1:50394 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1900820 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1900827 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1900845 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1900850 T4458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67601022 name:ZooKeeperConnection Watcher:127.0.0.1:50394/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1900851 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1900856 T4436 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1900867 T4436 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1900877 T4436 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1900883 T4436 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50397__kiw%2Fqj
   [junit4]   2> 1900886 T4436 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50397__kiw%2Fqj
   [junit4]   2> 1900898 T4436 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1900914 T4436 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1900921 T4436 oasc.Overseer.start Overseer (id=90290535195541507-127.0.0.1:50397__kiw%2Fqj-n_0000000000) starting
   [junit4]   2> 1900933 T4436 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1900945 T4460 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1900949 T4436 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1900961 T4436 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1900967 T4436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1900977 T4459 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1900994 T4436 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179
   [junit4]   2> 1901171 T4436 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/conf/
   [junit4]   2> 1901185 T4436 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1901187 T4436 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1901188 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1901189 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1901217 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 1901221 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1901226 T4463 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73e5941e name:ZooKeeperConnection Watcher:127.0.0.1:50394/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1901226 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1901231 T4436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1901239 T4436 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 1901709 T4436 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1901717 T4436 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50401
   [junit4]   2> 1901719 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1901720 T4436 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1901721 T4436 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1377724232367
   [junit4]   2> 1901722 T4436 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1377724232367/'
   [junit4]   2> 1901788 T4436 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1377724232367/solr.xml
   [junit4]   2> 1902085 T4436 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1902086 T4436 oasc.CoreContainer.<init> New CoreContainer 1098753712
   [junit4]   2> 1902087 T4436 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1377724232367/]
   [junit4]   2> 1902091 T4436 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1902092 T4436 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1902093 T4436 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1902094 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1902095 T4436 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1902096 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1902096 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1902097 T4436 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1902098 T4436 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1902100 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1902139 T4436 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1902141 T4436 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1902143 T4436 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50394/solr
   [junit4]   2> 1902144 T4436 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1902147 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1902154 T4474 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@48ccc248 name:ZooKeeperConnection Watcher:127.0.0.1:50394 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1902155 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1902162 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1902190 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1902196 T4476 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a359ec1 name:ZooKeeperConnection Watcher:127.0.0.1:50394/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1902197 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1902218 T4436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1903238 T4436 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50401__kiw%2Fqj
   [junit4]   2> 1903241 T4436 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50401__kiw%2Fqj
   [junit4]   2> 1903249 T4458 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1903249 T4463 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1903249 T4476 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1903260 T4436 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1377724232367
   [junit4]   2> 1903391 T4436 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1377724232367/conf/
   [junit4]   2> 1903399 T4436 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1903400 T4436 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1903401 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1903402 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1903934 T4436 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1903940 T4436 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50404
   [junit4]   2> 1903943 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1903943 T4436 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1903944 T4436 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552
   [junit4]   2> 1903945 T4436 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/'
   [junit4]   2> 1904004 T4436 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/solr.xml
   [junit4]   2> 1904138 T4436 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1904139 T4436 oasc.CoreContainer.<init> New CoreContainer 107424561
   [junit4]   2> 1904140 T4436 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/]
   [junit4]   2> 1904142 T4436 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1904143 T4436 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1904143 T4436 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1904144 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1904144 T4436 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1904145 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1904145 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1904146 T4436 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1904146 T4436 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1904147 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1904166 T4436 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1904167 T4436 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1904167 T4436 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50394/solr
   [junit4]   2> 1904168 T4436 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1904170 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1904174 T4488 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3561913 name:ZooKeeperConnection Watcher:127.0.0.1:50394 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1904175 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1904180 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1904204 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1904210 T4490 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56666937 name:ZooKeeperConnection Watcher:127.0.0.1:50394/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1904211 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1904227 T4436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1905238 T4436 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50404__kiw%2Fqj
   [junit4]   2> 1905241 T4436 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50404__kiw%2Fqj
   [junit4]   2> 1905251 T4490 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1905252 T4458 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1905252 T4476 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1905253 T4463 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1905270 T4436 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552
   [junit4]   2> 1905403 T4436 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/conf/
   [junit4]   2> 1905411 T4436 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1905412 T4436 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1905413 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1905413 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1905885 T4436 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1905891 T4436 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50407
   [junit4]   2> 1905893 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1905895 T4436 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1905896 T4436 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561
   [junit4]   2> 1905896 T4436 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/'
   [junit4]   2> 1905941 T4436 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/solr.xml
   [junit4]   2> 1906065 T4436 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1906067 T4436 oasc.CoreContainer.<init> New CoreContainer 1038160682
   [junit4]   2> 1906067 T4436 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/]
   [junit4]   2> 1906070 T4436 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1906071 T4436 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1906071 T4436 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1906072 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1906072 T4436 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1906073 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1906073 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1906074 T4436 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1906075 T4436 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1906075 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1906094 T4436 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1906095 T4436 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1906096 T4436 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50394/solr
   [junit4]   2> 1906097 T4436 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1906098 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1906104 T4502 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e711052 name:ZooKeeperConnection Watcher:127.0.0.1:50394 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1906105 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1906110 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1906135 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1906141 T4504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fd41225 name:ZooKeeperConnection Watcher:127.0.0.1:50394/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1906141 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1906156 T4436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1907167 T4436 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50407__kiw%2Fqj
   [junit4]   2> 1907170 T4436 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50407__kiw%2Fqj
   [junit4]   2> 1907180 T4490 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1907180 T4504 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1907181 T4476 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1907182 T4458 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1907183 T4463 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1907199 T4436 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561
   [junit4]   2> 1907405 T4436 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/conf/
   [junit4]   2> 1907416 T4436 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1907418 T4436 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1907418 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1907419 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1907926 T4436 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1907930 T4436 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:50410
   [junit4]   2> 1907932 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1907933 T4436 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1907933 T4436 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564
   [junit4]   2> 1907934 T4436 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/'
   [junit4]   2> 1907971 T4436 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/solr.xml
   [junit4]   2> 1908073 T4436 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 1908074 T4436 oasc.CoreContainer.<init> New CoreContainer 243819236
   [junit4]   2> 1908075 T4436 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/]
   [junit4]   2> 1908077 T4436 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1908078 T4436 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1908079 T4436 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1908079 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1908080 T4436 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1908080 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1908081 T4436 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1908081 T4436 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1908082 T4436 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1908082 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1908101 T4436 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1908102 T4436 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1908102 T4436 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50394/solr
   [junit4]   2> 1908103 T4436 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1908105 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1908109 T4516 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3004280c name:ZooKeeperConnection Watcher:127.0.0.1:50394 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1908110 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1908115 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1908136 T4436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1908142 T4518 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@184db4b name:ZooKeeperConnection Watcher:127.0.0.1:50394/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1908142 T4436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1908155 T4436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1909174 T4436 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50410__kiw%2Fqj
   [junit4]   2> 1909177 T4436 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50410__kiw%2Fqj
   [junit4]   2> 1909188 T4463 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1909189 T4476 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1909189 T4490 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1909191 T4504 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1909191 T4518 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1909192 T4458 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1909204 T4436 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564
   [junit4]   2> 1909385 T4436 oasc.CorePropertiesLocator.discoverUnder Found core conf in /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/conf/
   [junit4]   2> 1909400 T4436 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1909402 T4436 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1909403 T4436 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1909404 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1909429 T4436 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1909448 T4468 oasha.CollectionsHandler.handleCreateAction Creating Collection : numShards=2&name=nodes_used_collection&replicationFactor=2&action=CREATE&wt=javabin&version=2
   [junit4]   2> 1909451 T4458 oasc.DistributedQueue$LatchChildWatcher.process Watcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1909454 T4460 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 1909454 T4460 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "name":"nodes_used_collection",
   [junit4]   2> 	  "replicationFactor":"2"}
   [junit4]   2> 1910034 T4459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1910036 T4459 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: nodes_used_collection
   [junit4]   2> 1910036 T4459 oasc.Overseer$ClusterStateUpdater.createCollection Create collection nodes_used_collection with shards [shard1, shard2]
   [junit4]   2> 1910048 T4490 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1910048 T4476 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1910048 T4518 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1910048 T4504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1910048 T4458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1910048 T4463 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1910074 T4460 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 1910075 T4460 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica1 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:50407__kiw%2Fqj
   [junit4]   2> 1910076 T4460 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard1_replica2 as part of slice shard1 of collection nodes_used_collection on 127.0.0.1:50397__kiw%2Fqj
   [junit4]   2> 1910077 T4460 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica1 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:50404__kiw%2Fqj
   [junit4]   2> 1910079 T4460 oasc.OverseerCollectionProcessor.createCollection Creating shard nodes_used_collection_shard2_replica2 as part of slice shard2 of collection nodes_used_collection on 127.0.0.1:50410__kiw%2Fqj
   [junit4]   2> 1910092 T4499 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica1 state=down
   [junit4]   2> 1910092 T4454 oasc.ZkController.publish publishing core=nodes_used_collection_shard1_replica2 state=down
   [junit4]   2> 1910093 T4483 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica1 state=down
   [junit4]   2> 1910095 T4513 oasc.ZkController.publish publishing core=nodes_used_collection_shard2_replica2 state=down
   [junit4]   2> 1910098 T4499 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1910098 T4454 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1910100 T4483 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1910099 T4513 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1911565 T4459 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1911567 T4459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50397__kiw%2Fqj",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50397/_kiw/qj"}
   [junit4]   2> 1911577 T4459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard1_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50407__kiw%2Fqj",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50407/_kiw/qj"}
   [junit4]   2> 1911587 T4459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica2",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50410__kiw%2Fqj",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50410/_kiw/qj"}
   [junit4]   2> 1911597 T4459 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"nodes_used_collection_shard2_replica1",
   [junit4]   2> 	  "collection":"nodes_used_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:50404__kiw%2Fqj",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50404/_kiw/qj"}
   [junit4]   2> 1911608 T4463 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1911608 T4458 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1911608 T4504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1911609 T4476 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1911609 T4490 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1911609 T4518 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1912101 T4499 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica1
   [junit4]   2> 1912102 T4454 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard1_replica2
   [junit4]   2> 1912105 T4454 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2
   [junit4]   2> 1912105 T4454 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1912106 T4483 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica1
   [junit4]   2> 1912105 T4513 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for nodes_used_collection_shard2_replica2
   [junit4]   2> 1912109 T4513 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2
   [junit4]   2> 1912110 T4513 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1912106 T4499 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard1_replica1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1
   [junit4]   2> 1912111 T4499 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1912110 T4454 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1912113 T4454 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1912109 T4483 oasc.CoreContainer.create Creating SolrCore 'nodes_used_collection_shard2_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1
   [junit4]   2> 1912114 T4483 oasc.ZkController.createCollectionZkNode Check for collection zkNode:nodes_used_collection
   [junit4]   2> 1912113 T4499 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1912112 T4513 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1912116 T4513 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1912117 T4483 oasc.ZkController.createCollectionZkNode Creating collection in ZooKeeper:nodes_used_collection
   [junit4]   2> 1912117 T4454 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1912115 T4499 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1912118 T4454 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1912117 T4483 oasc.ZkController.getConfName Looking for collection configName
   [junit4]   2> 1912121 T4513 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1912123 T4513 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1912124 T4499 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1912125 T4499 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1912126 T4483 oasc.ZkController.getConfName Only one config set found in zk - using it:conf1
   [junit4]   2> 1912127 T4483 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection
   [junit4]   2> 1912132 T4499 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1912132 T4513 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1912133 T4454 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1912135 T4483 oasc.ZkController.readConfigName Load collection config from:/collections/nodes_used_collection
   [junit4]   2> 1912138 T4499 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1/'
   [junit4]   2> 1912139 T4454 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2/'
   [junit4]   2> 1912141 T4483 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1/'
   [junit4]   2> 1912144 T4513 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2/'
   [junit4]   2> 1912346 T4499 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1912479 T4513 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1912484 T4454 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1912508 T4483 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1912691 T4513 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1912704 T4513 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1912707 T4499 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1912719 T4499 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1912723 T4483 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1912728 T4483 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1912748 T4513 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 1912750 T4454 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1912757 T4454 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1912778 T4454 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 1912783 T4499 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 1912798 T4483 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 1915166 T4499 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1915200 T4499 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1915215 T4499 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1915236 T4483 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1915242 T4454 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1915270 T4499 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915272 T4483 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1915277 T4513 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1915279 T4499 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915275 T4454 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1915287 T4499 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915292 T4513 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1915298 T4483 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1915299 T4454 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1915304 T4499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915305 T4499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915305 T4499 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915313 T4513 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1915314 T4499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915317 T4499 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915317 T4499 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1915318 T4499 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1/, dataDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1/data/
   [junit4]   2> 1915320 T4499 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a4d72ef
   [junit4]   2> 1915324 T4499 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1/data
   [junit4]   2> 1915325 T4499 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1/data/index/
   [junit4]   2> 1915325 T4499 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica1] Solr index directory '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1915327 T4499 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1377724236561/nodes_used_collection_shard1_replica1/data/index
   [junit4]   2> 1915329 T4499 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=34.859375, floorSegmentMB=2.1748046875, forceMergeDeletesPctAllowed=14.516550202218715, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7523145911573198
   [junit4]   2> 1915334 T4499 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1449c32e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16dfc93b),segFN=segments_1,generation=1}
   [junit4]   2> 1915335 T4499 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915358 T4499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1915361 T4499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915369 T4499 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915372 T4499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915375 T4499 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915385 T4499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1915387 T4499 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915383 T4454 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915393 T4483 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915393 T4499 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915401 T4499 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1915403 T4499 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1915404 T4499 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1915405 T4499 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1915405 T4483 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915405 T4499 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1915414 T4499 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1915415 T4499 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1915416 T4483 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915411 T4454 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915420 T4483 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915421 T4483 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915421 T4483 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915426 T4513 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915433 T4483 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915433 T4483 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915434 T4483 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1915435 T4483 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1/data/
   [junit4]   2> 1915435 T4483 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a4d72ef
   [junit4]   2> 1915434 T4454 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915438 T4483 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1/data
   [junit4]   2> 1915439 T4483 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1/data/index/
   [junit4]   2> 1915439 T4483 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1915441 T4483 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1377724234552/nodes_used_collection_shard2_replica1/data/index
   [junit4]   2> 1915442 T4483 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=34.859375, floorSegmentMB=2.1748046875, forceMergeDeletesPctAllowed=14.516550202218715, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7523145911573198
   [junit4]   2> 1915441 T4499 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1915446 T4513 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1915450 T4454 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915450 T4454 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915451 T4454 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915452 T4499 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1915454 T4499 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1915455 T4499 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44625116582377033]
   [junit4]   2> 1915456 T4499 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1449c32e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16dfc93b),segFN=segments_1,generation=1}
   [junit4]   2> 1915459 T4483 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4e4d9f3c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@15d6de00),segFN=segments_1,generation=1}
   [junit4]   2> 1915460 T4483 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915459 T4499 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915461 T4499 oass.SolrIndexSearcher.<init> Opening Searcher@3982846c main
   [junit4]   2> 1915458 T4513 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915460 T4454 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915465 T4454 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915466 T4454 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1915466 T4454 oasc.SolrCore.<init> [nodes_used_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2/data/
   [junit4]   2> 1915467 T4454 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a4d72ef
   [junit4]   2> 1915468 T4454 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2/data
   [junit4]   2> 1915468 T4454 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2/data/index/
   [junit4]   2> 1915469 T4454 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1915469 T4483 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1915471 T4483 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915471 T4483 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915474 T4483 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915477 T4483 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915477 T4483 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1915478 T4483 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915479 T4483 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915479 T4483 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1915480 T4483 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1915481 T4524 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica1] Registered new searcher Searcher@3982846c main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1915482 T4483 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1915483 T4483 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1915483 T4483 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1915484 T4483 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1915485 T4483 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1915470 T4454 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1377724231179/nodes_used_collection_shard1_replica2/data/index
   [junit4]   2> 1915487 T4454 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=34.859375, floorSegmentMB=2.1748046875, forceMergeDeletesPctAllowed=14.516550202218715, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7523145911573198
   [junit4]   2> 1915485 T4499 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica1
   [junit4]   2> 1915488 T4499 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica1 address:http://127.0.0.1:50407/_kiw/qj collection:nodes_used_collection shard:shard1
   [junit4]   2> 1915475 T4513 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915489 T4513 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915490 T4513 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1915489 T4454 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7c0e4865 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3eaef3da),segFN=segments_1,generation=1}
   [junit4]   2> 1915491 T4454 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915489 T4499 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1915497 T4454 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1915498 T4454 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915499 T4454 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915499 T4454 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915500 T4454 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915501 T4454 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1915502 T4454 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915503 T4454 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915503 T4454 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1915488 T4483 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1915504 T4454 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1915507 T4454 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1915508 T4454 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1915509 T4454 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1915511 T4454 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1915512 T4454 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1915504 T4513 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1915513 T4513 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1915514 T4513 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1915515 T4513 oasc.SolrCore.<init> [nodes_used_collection_shard2_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2/data/
   [junit4]   2> 1915515 T4513 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a4d72ef
   [junit4]   2> 1915516 T4454 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1915517 T4513 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2/data
   [junit4]   2> 1915518 T4513 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2/data/index/
   [junit4]   2> 1915518 T4513 oasc.SolrCore.initIndex WARN [nodes_used_collection_shard2_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1915519 T4513 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/nodes_used_collection_shard2_replica2/data/index
   [junit4]   2> 1915519 T4513 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=32, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=34.859375, floorSegmentMB=2.1748046875, forceMergeDeletesPctAllowed=14.516550202218715, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7523145911573198
   [junit4]   2> 1915521 T4513 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@a37f250 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7f22c554),segFN=segments_1,generation=1}
   [junit4]   2> 1915521 T4513 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915527 T4454 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1915527 T4454 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1915529 T4454 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44625116582377033]
   [junit4]   2> 1915528 T4513 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1915533 T4513 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915535 T4513 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1915528 T4499 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard1/election
   [junit4]   2> 1915536 T4513 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915537 T4513 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1915537 T4513 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1915539 T4513 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915539 T4513 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1915540 T4513 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1915541 T4513 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1915535 T4454 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@7c0e4865 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3eaef3da),segFN=segments_1,generation=1}
   [junit4]   2> 1915543 T4454 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915543 T4454 oass.SolrIndexSearcher.<init> Opening Searcher@5a0d0f09 main
   [junit4]   2> 1915547 T4513 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1915548 T4513 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1915550 T4513 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1915535 T4483 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1915551 T4483 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1915551 T4454 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard1_replica2
   [junit4]   2> 1915555 T4454 oasc.ZkController.register Register replica - core:nodes_used_collection_shard1_replica2 address:http://127.0.0.1:50397/_kiw/qj collection:nodes_used_collection shard:shard1
   [junit4]   2> 1915555 T4454 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1915549 T4525 oasc.SolrCore.registerSearcher [nodes_used_collection_shard1_replica2] Registered new searcher Searcher@5a0d0f09 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1915555 T4483 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44625116582377033]
   [junit4]   2> 1915552 T4513 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1915566 T4513 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1915569 T4513 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1915577 T4483 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4e4d9f3c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@15d6de00),segFN=segments_1,generation=1}
   [junit4]   2> 1915578 T4483 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915579 T4513 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1915580 T4513 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1915579 T4483 oass.SolrIndexSearcher.<init> Opening Searcher@77a1b4cb main
   [junit4]   2> 1915582 T4513 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=26, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.44625116582377033]
   [junit4]   2> 1915587 T4513 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@a37f250 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7f22c554),segFN=segments_1,generation=1}
   [junit4]   2> 1915587 T4526 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica1] Registered new searcher Searcher@77a1b4cb main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1915588 T4513 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1915590 T4513 oass.SolrIndexSearcher.<init> Opening Searcher@68320fce main
   [junit4]   2> 1915593 T4483 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica1
   [junit4]   2> 1915594 T4483 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica1 address:http://127.0.0.1:50404/_kiw/qj collection:nodes_used_collection shard:shard2
   [junit4]   2> 1915594 T4483 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1915598 T4527 oasc.SolrCore.registerSearcher [nodes_used_collection_shard2_replica2] Registered new searcher Searcher@68320fce main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1915603 T4513 oasc.CoreContainer.registerCore registering core: nodes_used_collection_shard2_replica2
   [junit4]   2> 1915604 T4513 oasc.ZkController.register Register replica - core:nodes_used_collection_shard2_replica2 address:http://127.0.0.1:50410/_kiw/qj collection:nodes_used_collection shard:shard2
   [junit4]   2> 1915610 T4513 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1915617 T4499 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1915619 T4483 oascc.SolrZkClient.makePath makePath: /collections/nodes_used_collection/leader_elect/shard2/election
   [junit4]   2> 1915630 T4499 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1915632 T4499 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1915632 T4499 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:50407/_kiw/qj/nodes_used_collection_shard1_replica1/
   [junit4]   2> 1915633 T4499 oasu.PeerSync.sync PeerSync: core=nodes_used_collection_shard1_replica1 url=http://127.0.0.1:504

[...truncated too long message...]

973696 T4436 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1973697 T4436 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1973697 T4436 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1973698 T4436 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1973699 T4436 oasc.SolrCore.closeSearcher [awholynewcollection_1_shard3_replica5] Closing main searcher on request.
   [junit4]   2> 1973700 T4436 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1973700 T4436 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/awholynewcollection_1_shard3_replica5/data/index [CachedDir<<refCount=0;path=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/awholynewcollection_1_shard3_replica5/data/index;done=false>>]
   [junit4]   2> 1973701 T4436 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/awholynewcollection_1_shard3_replica5/data/index
   [junit4]   2> 1973701 T4436 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/awholynewcollection_1_shard3_replica5/data [CachedDir<<refCount=0;path=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/awholynewcollection_1_shard3_replica5/data;done=false>>]
   [junit4]   2> 1973702 T4436 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1377724238564/awholynewcollection_1_shard3_replica5/data
   [junit4]   2> 1973704 T4704 oasc.Overseer$ClusterStateUpdater.amILeader According to ZK I (id=90290535195541516-127.0.0.1:50410__kiw%2Fqj-n_0000000004) am no longer a leader.
   [junit4]   2> 1973706 T4518 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1973706 T4518 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1973707 T4518 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1973722 T4436 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/_kiw/qj,null}
   [junit4]   2> 1973787 T4436 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1973790 T4436 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:50394 50394
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=19679AFCD04CEFCA -Dtests.slow=true -Dtests.locale=ar_TN -Dtests.timezone=Africa/Khartoum -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   74.2s | CollectionsAPIDistributedZkTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: awholynewcollection_1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([19679AFCD04CEFCA:988114E4A7138FF6]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:424)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:264)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1571)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1585)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:481)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:152)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 1973925 T4436 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=DefaultSimilarity, locale=ar_TN, timezone=Africa/Khartoum
   [junit4]   2> NOTE: Mac OS X 10.8.4 x86_64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=2,threads=6,free=122248256,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [MoreLikeThisHandlerTest, IndexSchemaTest, TestCollationField, HdfsUnloadDistributedZkTest, SortByFunctionTest, BasicZkTest, TestSolrXmlPersistence, TestPhraseSuggestions, UnloadDistributedZkTest, TestNRTOpen, AddSchemaFieldsUpdateProcessorFactoryTest, TermVectorComponentDistributedTest, TestRTGBase, TestReplicationHandler, TestSolrXmlPersistor, ScriptEngineTest, PrimUtilsTest, TestSchemaVersionResource, HdfsBasicDistributedZk2Test, SolrCoreTest, TestRealTimeGet, TestRandomFaceting, TestCopyFieldCollectionResource, HdfsLockFactoryTest, TestRangeQuery, PluginInfoTest, BasicDistributedZkTest, RequiredFieldsTest, HdfsCollectionsAPIDistributedZkTest, TestSchemaResource, PathHierarchyTokenizerFactoryTest, ExternalFileFieldSortTest, SystemInfoHandlerTest, TestJoin, BlockCacheTest, SuggesterFSTTest, DirectSolrSpellCheckerTest, LeaderElectionTest, TestLMDirichletSimilarityFactory, TestSystemIdResolver, TestArbitraryIndexDir, ZkControllerTest, DistanceFunctionTest, DefaultValueUpdateProcessorTest, OverseerCollectionProcessorTest, StressHdfsTest, SchemaVersionSpecificBehaviorTest, TestReload, TestDFRSimilarityFactory, SOLR749Test, SampleTest, TestStressLucene, SimplePostToolTest, TestSolrDeletionPolicy1, FieldMutatingUpdateProcessorTest, ZkCLITest, AnalysisAfterCoreReloadTest, TestAtomicUpdateErrorCases, CoreContainerCoreInitFailuresTest, TestGroupingSearch, HighlighterConfigTest, TestFoldingMultitermQuery, LukeRequestHandlerTest, TestPerFieldSimilarity, HighlighterTest, TestManagedSchema, CopyFieldTest, ResponseLogComponentTest, IndexBasedSpellCheckerTest, TestMergePolicyConfig, DocValuesMultiTest, TestDistributedSearch, RegexBoostProcessorTest, WordBreakSolrSpellCheckerTest, TestDefaultSimilarityFactory, IndexSchemaRuntimeFieldTest, TestSolrJ, TestCSVLoader, SignatureUpdateProcessorFactoryTest, ResourceLoaderTest, ClusterStateTest, TestWordDelimiterFilterFactory, TestSerializedLuceneMatchVersion, SolrCoreCheckLockOnStartupTest, TestInfoStreamLogging, DirectUpdateHandlerOptimizeTest, MultiTermTest, FileBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestSolrXml, SliceStateUpdateTest, TestNumberUtils, DirectUpdateHandlerTest, DateFieldTest, SearchHandlerTest, LegacyHTMLStripCharFilterTest, TestRecovery, TestFastWriter, TestShardHandlerFactory, TestMultiCoreConfBootstrap, TestNoOpRegenerator, TestDefaultSearchFieldResource, SliceStateTest, TestFieldTypeResource, TestCSVResponseWriter, FastVectorHighlighterTest, QueryParsingTest, TestSuggestSpellingConverter, JSONWriterTest, TestFunctionQuery, HdfsChaosMonkeySafeLeaderTest, TestFaceting, TestDocumentBuilder, TestQueryUtils, ShardRoutingCustomTest, TestStressVersions, TestSchemaNameResource, TestWriterPerf, TestSolrQueryParser, SynonymTokenizerTest, TestMaxScoreQueryParser, BlockDirectoryTest, TestConfig, TestStressReorder, TestSolrDeletionPolicy2, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, ShardSplitTest, TestCoreContainer, TestSolrCoreProperties, AutoCommitTest, BasicDistributedZk2Test, BadIndexSchemaTest, NoCacheHeaderTest, TestQuerySenderListener, HdfsDirectoryTest, ZkSolrClientTest, StatsComponentTest, SolrCmdDistributorTest, LeaderElectionIntegrationTest, AssignTest, CoreAdminCreateDiscoverTest, TestIBSimilarityFactory, TimeZoneUtilsTest, OutputWriterTest, SuggesterTSTTest, PolyFieldTest, TestLuceneMatchVersion, TestZkChroot, HdfsBasicDistributedZkTest, NotRequiredUniqueKeyTest, SolrPluginUtilsTest, TestPHPSerializedResponseWriter, TestFieldResource, AliasIntegrationTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed in 74.24s, 1 test, 1 error <<< FAILURES!

[...truncated 507 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:389: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:369: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:450: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1230: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:873: There were test failures: 324 suites, 1406 tests, 1 error, 40 ignored (5 assumptions)

Total time: 96 minutes 59 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure