You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/01/10 19:05:18 UTC

[JENKINS] Lucene-Solr-trunk-Linux (32bit/jdk1.8.0-ea-b121) - Build # 9043 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/9043/
Java: 32bit/jdk1.8.0-ea-b121 -client -XX:+UseG1GC

1 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([D7B72BD6048F420F:5651A5CE73D02233]:0)
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.junit.Assert.assertFalse(Assert.java:79)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.deleteCollectionWithDownNodes(CollectionsAPIDistributedZkTest.java:390)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:203)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)




Build Log:
[...truncated 10259 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 585164 T566 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 585167 T566 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./solrtest-CollectionsAPIDistributedZkTest-1389376367732
   [junit4]   2> 585167 T566 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 585168 T567 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 585268 T566 oasc.ZkTestServer.run start zk server on port:57673
   [junit4]   2> 585269 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585379 T573 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3cb929 name:ZooKeeperConnection Watcher:127.0.0.1:57673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585380 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585380 T566 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 585388 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585390 T575 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15a056c name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585391 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585392 T566 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 585397 T566 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 585402 T566 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 585405 T566 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 585410 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 585411 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 585418 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 585419 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 585426 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 585427 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 585432 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 585433 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 585437 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 585438 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 585442 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 585443 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 585447 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 585448 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 585453 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 585454 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 585458 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 585459 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 585464 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 585465 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 585469 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 585470 T566 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> Using new style solr.xml
   [junit4]   2> 585478 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585481 T577 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@114872e name:ZooKeeperConnection Watcher:127.0.0.1:57673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585481 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585482 T566 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 585486 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585488 T579 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@719e0b name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585489 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585491 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 585492 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 585499 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 585500 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
   [junit4]   2> 585506 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 585507 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 585513 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 585514 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 585519 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 585520 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
   [junit4]   2> 585525 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 585526 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
   [junit4]   2> 585531 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 585532 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 585535 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 585535 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 585538 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 585538 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 585541 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 585541 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 585543 T566 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 585544 T566 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
   [junit4]   2> 585625 T566 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 585629 T566 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:36598
   [junit4]   2> 585629 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 585630 T566 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 585630 T566 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112
   [junit4]   2> 585630 T566 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/'
   [junit4]   2> 585643 T566 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solr.xml
   [junit4]   2> 585653 T566 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 585653 T566 oasc.CoreContainer.<init> New CoreContainer 18461037
   [junit4]   2> 585654 T566 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/]
   [junit4]   2> 585654 T566 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 585655 T566 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 585655 T566 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 585655 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 585655 T566 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 585656 T566 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 585656 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 585656 T566 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 585657 T566 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 585659 T566 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 585659 T566 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 585659 T566 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 585660 T566 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57673/solr
   [junit4]   2> 585660 T566 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 585661 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585663 T590 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d0307 name:ZooKeeperConnection Watcher:127.0.0.1:57673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585664 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585666 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585667 T592 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f8b0e5 name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585667 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585668 T566 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 585671 T566 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 585673 T566 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 585675 T566 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36598_
   [junit4]   2> 585675 T566 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36598_
   [junit4]   2> 585677 T566 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 585679 T566 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 585682 T566 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 585684 T566 oasc.Overseer.start Overseer (id=91054169635880965-127.0.0.1:36598_-n_0000000000) starting
   [junit4]   2> 585686 T566 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 585689 T594 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 585690 T566 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 585692 T566 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 585694 T566 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 585696 T593 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 585698 T566 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112
   [junit4]   2> 585699 T566 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/conf/
   [junit4]   2> 585726 T566 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 585726 T566 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 585727 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 585728 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585731 T597 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c9e887 name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585731 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585733 T566 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 585735 T566 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 585838 T566 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 585841 T566 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:56735
   [junit4]   2> 585843 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 585843 T566 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 585844 T566 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300
   [junit4]   2> 585844 T566 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/'
   [junit4]   2> 585859 T566 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solr.xml
   [junit4]   2> 585868 T566 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 585869 T566 oasc.CoreContainer.<init> New CoreContainer 5448216
   [junit4]   2> 585869 T566 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/]
   [junit4]   2> 585870 T566 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 585870 T566 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 585870 T566 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 585871 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 585871 T566 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 585871 T566 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 585872 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 585872 T566 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 585872 T566 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 585874 T566 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 585875 T566 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 585875 T566 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 585875 T566 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57673/solr
   [junit4]   2> 585876 T566 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 585876 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585878 T608 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@93f7a5 name:ZooKeeperConnection Watcher:127.0.0.1:57673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585879 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585881 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 585882 T610 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16c4dd1 name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 585883 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 585887 T566 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 586890 T566 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56735_
   [junit4]   2> 586891 T566 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:56735_
   [junit4]   2> 586893 T597 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 586894 T592 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 586894 T610 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 586897 T566 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300
   [junit4]   2> 586898 T566 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/conf/
   [junit4]   2> 586933 T566 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 586934 T566 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 586934 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 587016 T566 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 587018 T566 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:34287
   [junit4]   2> 587018 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 587019 T566 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 587019 T566 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500
   [junit4]   2> 587019 T566 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/'
   [junit4]   2> 587035 T566 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solr.xml
   [junit4]   2> 587046 T566 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 587047 T566 oasc.CoreContainer.<init> New CoreContainer 1272154
   [junit4]   2> 587047 T566 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/]
   [junit4]   2> 587048 T566 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 587048 T566 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 587049 T566 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 587049 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 587049 T566 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 587050 T566 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 587050 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 587050 T566 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 587051 T566 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 587053 T566 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 587053 T566 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 587054 T566 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 587054 T566 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57673/solr
   [junit4]   2> 587055 T566 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 587055 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 587057 T622 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1400610 name:ZooKeeperConnection Watcher:127.0.0.1:57673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 587057 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 587059 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 587061 T624 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e081f6 name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 587061 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 587064 T566 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 588066 T566 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34287_
   [junit4]   2> 588068 T566 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34287_
   [junit4]   2> 588072 T610 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 588073 T597 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 588073 T624 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 588073 T592 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 588077 T566 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500
   [junit4]   2> 588078 T566 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/conf/
   [junit4]   2> 588114 T566 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 588115 T566 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 588115 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 588202 T566 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 588205 T566 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41895
   [junit4]   2> 588206 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 588206 T566 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 588207 T566 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1389376370681
   [junit4]   2> 588207 T566 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1389376370681/'
   [junit4]   2> 588227 T566 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1389376370681/solr.xml
   [junit4]   2> 588250 T566 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 588251 T566 oasc.CoreContainer.<init> New CoreContainer 15475940
   [junit4]   2> 588251 T566 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1389376370681/]
   [junit4]   2> 588252 T566 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 588252 T566 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 588253 T566 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 588253 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 588254 T566 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 588254 T566 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 588255 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 588255 T566 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 588255 T566 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 588258 T566 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 588259 T566 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 588259 T566 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 588260 T566 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57673/solr
   [junit4]   2> 588260 T566 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 588261 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 588263 T636 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13ade3d name:ZooKeeperConnection Watcher:127.0.0.1:57673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 588264 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 588267 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 588270 T638 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ccfcf5 name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 588270 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 588311 T566 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 589316 T566 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41895_
   [junit4]   2> 589319 T566 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41895_
   [junit4]   2> 589325 T624 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 589325 T597 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 589326 T610 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 589326 T592 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 589326 T638 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 589334 T566 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1389376370681
   [junit4]   2> 589337 T566 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1389376370681/conf/
   [junit4]   2> 589388 T566 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 589389 T566 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 589389 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 589483 T566 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 589485 T566 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:47909
   [junit4]   2> 589486 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 589486 T566 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 589486 T566 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955
   [junit4]   2> 589487 T566 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/'
   [junit4]   2> 589503 T566 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solr.xml
   [junit4]   2> 589513 T566 oasc.ConfigSolrXml.<init> Config-defined core root directory: 
   [junit4]   2> 589513 T566 oasc.CoreContainer.<init> New CoreContainer 24111803
   [junit4]   2> 589514 T566 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/]
   [junit4]   2> 589514 T566 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 589515 T566 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 589515 T566 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 589515 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 589516 T566 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 589516 T566 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 589516 T566 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 589517 T566 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 589517 T566 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 589520 T566 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 589521 T566 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 589521 T566 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 589521 T566 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:57673/solr
   [junit4]   2> 589522 T566 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 589522 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 589525 T650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19b535c name:ZooKeeperConnection Watcher:127.0.0.1:57673 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 589526 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 589529 T566 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 589530 T652 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a4a9 name:ZooKeeperConnection Watcher:127.0.0.1:57673/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 589531 T566 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 589534 T566 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 590537 T566 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47909_
   [junit4]   2> 590539 T566 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47909_
   [junit4]   2> 590545 T597 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 590545 T610 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 590546 T652 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 590546 T638 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 590546 T592 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 590546 T624 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 590553 T566 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955
   [junit4]   2> 590555 T566 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/conf/
   [junit4]   2> 590597 T566 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 590598 T566 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 590599 T566 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 590610 T603 oasha.CollectionsHandler.handleCreateAction Creating Collection : replicationFactor=2&collection.configName=conf1&router.field=myOwnField&name=solrj_collection&router.name=compositeId&action=CREATE&numShards=2&wt=javabin&version=2
   [junit4]   2> 590611 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590613 T594 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 590614 T594 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 590614 T594 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
   [junit4]   2> 590615 T594 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
   [junit4]   2> 590618 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590620 T593 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 590620 T593 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
   [junit4]   2> 590620 T593 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
   [junit4]   2> 590623 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590623 T638 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590624 T610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590624 T592 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590624 T624 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590624 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590624 T652 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590719 T594 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 590719 T594 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:36598_
   [junit4]   2> 590720 T594 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:47909_
   [junit4]   2> 590721 T594 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:56735_
   [junit4]   2> 590722 T594 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:34287_
   [junit4]   2> 590723 T646 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
   [junit4]   2> 590723 T605 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
   [junit4]   2> 590723 T587 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
   [junit4]   2> 590724 T619 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
   [junit4]   2> 590725 T605 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 590724 T646 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 590724 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590726 T587 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 590726 T619 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 590729 T593 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 590730 T593 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47909",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47909_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 590734 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590736 T593 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:56735",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:56735_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 590738 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590740 T593 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36598",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36598_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 590743 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590744 T593 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34287",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34287_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 590746 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 590850 T638 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590851 T592 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590850 T610 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590850 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590850 T652 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 590850 T624 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 591725 T605 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
   [junit4]   2> 591725 T646 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
   [junit4]   2> 591727 T587 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
   [junit4]   2> 591727 T619 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
   [junit4]   2> 591729 T587 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1
   [junit4]   2> 591728 T646 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2
   [junit4]   2> 591730 T646 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 591728 T605 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1
   [junit4]   2> 591731 T605 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 591729 T587 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 591729 T619 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2
   [junit4]   2> 591733 T619 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 591733 T605 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 591734 T605 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 591731 T646 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 591734 T619 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 591733 T587 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 591737 T587 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 591737 T605 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1/'
   [junit4]   2> 591736 T619 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 591736 T646 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 591740 T587 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1/'
   [junit4]   2> 591741 T619 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2/'
   [junit4]   2> 591743 T646 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2/'
   [junit4]   2> 591814 T605 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 591819 T619 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 591819 T646 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 591838 T587 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 591841 T605 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 591842 T646 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 591843 T646 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 591844 T605 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 591844 T619 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 591846 T619 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 591850 T587 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 591852 T587 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 591855 T646 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
   [junit4]   2> 591856 T619 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
   [junit4]   2> 591856 T605 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
   [junit4]   2> 591859 T587 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
   [junit4]   2> 592033 T587 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 592037 T587 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 592039 T587 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 592050 T587 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592053 T587 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592055 T587 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592056 T587 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592059 T587 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592059 T587 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592060 T587 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592060 T587 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592061 T587 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 592061 T587 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1/data/
   [junit4]   2> 592062 T587 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@aeedbf
   [junit4]   2> 592063 T587 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1/data
   [junit4]   2> 592063 T587 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1/data/index/
   [junit4]   2> 592064 T587 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 592064 T587 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1389376368112/solrj_collection_shard1_replica1/data/index
   [junit4]   2> 592064 T587 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=93.6259765625, floorSegmentMB=0.798828125, forceMergeDeletesPctAllowed=9.328922708796675, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 592065 T587 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@d4ad28 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a97776),segFN=segments_1,generation=1}
   [junit4]   2> 592066 T587 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 592069 T587 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 592070 T587 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 592070 T587 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 592070 T587 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592071 T587 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592071 T587 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 592071 T587 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592072 T587 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592072 T587 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 592073 T587 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 592073 T587 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 592074 T587 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 592074 T587 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 592075 T587 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 592075 T587 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 592076 T587 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 592076 T587 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 592078 T587 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 592080 T587 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 592080 T587 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 592081 T587 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.27348538300142144]
   [junit4]   2> 592081 T587 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@d4ad28 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a97776),segFN=segments_1,generation=1}
   [junit4]   2> 592082 T587 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 592082 T587 oass.SolrIndexSearcher.<init> Opening Searcher@11fd5ca main
   [junit4]   2> 592086 T658 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@11fd5ca main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 592086 T619 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 592098 T587 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
   [junit4]   2> 592098 T587 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:36598 collection:solrj_collection shard:shard1
   [junit4]   2> 592100 T587 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
   [junit4]   2> 592101 T619 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 592104 T619 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 592105 T646 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 592106 T587 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 592110 T646 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 592111 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 592111 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 592111 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 592112 T587 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=179999
   [junit4]   2> 592112 T646 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 592112 T605 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 592113 T593 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 592117 T605 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 592119 T592 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 592120 T605 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 592129 T619 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592130 T646 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592133 T619 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592135 T605 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592136 T619 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592137 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592138 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592138 T619 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592138 T646 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592139 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592139 T619 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592140 T619 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 592140 T605 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 592140 T619 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2/data/
   [junit4]   2> 592141 T619 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@aeedbf
   [junit4]   2> 592141 T646 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592142 T619 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2/data
   [junit4]   2> 592142 T619 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2/data/index/
   [junit4]   2> 592143 T619 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 592143 T646 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592143 T646 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592143 T619 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1389376369500/solrj_collection_shard2_replica2/data/index
   [junit4]   2> 592143 T605 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592144 T619 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=93.6259765625, floorSegmentMB=0.798828125, forceMergeDeletesPctAllowed=9.328922708796675, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 592143 T646 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592145 T619 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1e2626c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@717710),segFN=segments_1,generation=1}
   [junit4]   2> 592145 T605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592146 T605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592146 T646 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592146 T646 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592145 T619 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 592147 T646 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 592146 T605 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 592147 T646 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2/data/
   [junit4]   2> 592148 T646 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@aeedbf
   [junit4]   2> 592148 T605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 592149 T605 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 592149 T646 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2/data
   [junit4]   2> 592149 T605 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 592149 T646 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2/data/index/
   [junit4]   2> 592150 T605 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1/data/
   [junit4]   2> 592150 T646 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 592150 T605 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@aeedbf
   [junit4]   2> 592151 T646 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/solrj_collection_shard1_replica2/data/index
   [junit4]   2> 592151 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 592152 T605 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1/data
   [junit4]   2> 592152 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 592152 T646 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=93.6259765625, floorSegmentMB=0.798828125, forceMergeDeletesPctAllowed=9.328922708796675, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 592152 T619 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 592152 T605 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1/data/index/
   [junit4]   2> 592153 T646 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@107004 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1689af9),segFN=segments_1,generation=1}
   [junit4]   2> 592153 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592154 T646 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 592153 T605 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 592154 T619 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592155 T605 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1389376368300/solrj_collection_shard2_replica1/data/index
   [junit4]   2> 592155 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 592156 T605 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=34, maxMergeAtOnceExplicit=43, maxMergedSegmentMB=93.6259765625, floorSegmentMB=0.798828125, forceMergeDeletesPctAllowed=9.328922708796675, segmentsPerTier=45.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 592156 T619 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592157 T619 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592157 T619 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 592157 T605 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1cc2108 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38323b),segFN=segments_1,generation=1}
   [junit4]   2> 592158 T619 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 592158 T605 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 592159 T619 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 592159 T619 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 592159 T619 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 592159 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 592160 T619 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 592160 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 592160 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 592161 T646 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 592161 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 592161 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592162 T619 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 592162 T646 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592163 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 592163 T646 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592163 T619 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 592164 T646 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592164 T605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 592164 T646 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 592165 T605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 592165 T619 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 592165 T646 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 592165 T605 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 592166 T619 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 592166 T646 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 592167 T646 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 592166 T605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592167 T646 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 592167 T619 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.27348538300142144]
   [junit4]   2> 592168 T646 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 592169 T619 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@1e2626c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@717710),segFN=segments_1,generation=1}
   [junit4]   2> 592169 T619 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 592168 T605 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 592169 T619 oass.SolrIndexSearcher.<init> Opening Searcher@14f47de main
   [junit4]   2> 592169 T646 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 592170 T605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 592171 T646 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 592171 T605 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592172 T605 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 592172 T646 oasc.RequestHandlers.initHandlersFromConfig created /admin/fileedit: solr.admin.EditFileRequestHandler
   [junit4]   2> 592172 T605 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 592173 T605 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 592174 T646 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 592173 T660 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica2] Registered new searcher Searcher@14f47de main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 592174 T605 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 592175 T605 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 592175 T605 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 592176 T646 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 592176 T605 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 592176 T646 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 592177 T605 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 592178 T619 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica2
   [junit4]   2> 592178 T646 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSize

[...truncated too long message...]

re/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1389376370681/nodes_used_collection_shard2_replica2/data
   [junit4]   2> 682525 T742 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 682525 T652 oascc.ZkStateReader$3.process Updating live nodes... (1)
   [junit4]   2> 682547 T566 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 682598 T566 oasc.ChaosMonkey.monkeyLog monkey: stop shard! 47909
   [junit4]   2> 682599 T566 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=24111803
   [junit4]   2> 682599 T566 oasc.ZkController.publish publishing core=acollectionafterbaddelete_shard1_replica2 state=down
   [junit4]   2> 682601 T652 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 682601 T566 oasc.SolrCore.close [acollectionafterbaddelete_shard1_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@8990ae
   [junit4]   2> 682602 T884 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 682603 T884 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47909",
   [junit4]   2> 	  "core":"acollectionafterbaddelete_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47909_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"acollectionafterbaddelete",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 682605 T652 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 682615 T566 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 682615 T566 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 682616 T566 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 682616 T566 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 682616 T566 oasc.SolrCore.closeSearcher [acollectionafterbaddelete_shard1_replica2] Closing main searcher on request.
   [junit4]   2> 682617 T566 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 682617 T566 oasc.CachingDirectoryFactory.closeCacheValue looking to close ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/acollectionafterbaddelete_shard1_replica2/data/index [CachedDir<<refCount=0;path=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/acollectionafterbaddelete_shard1_replica2/data/index;done=false>>]
   [junit4]   2> 682618 T566 oasc.CachingDirectoryFactory.close Closing directory: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/acollectionafterbaddelete_shard1_replica2/data/index
   [junit4]   2> 682618 T566 oasc.CachingDirectoryFactory.closeCacheValue looking to close ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/acollectionafterbaddelete_shard1_replica2/data [CachedDir<<refCount=0;path=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/acollectionafterbaddelete_shard1_replica2/data;done=false>>]
   [junit4]   2> 682618 T566 oasc.CachingDirectoryFactory.close Closing directory: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1389376371955/acollectionafterbaddelete_shard1_replica2/data
   [junit4]   2> 682642 T566 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 682696 T566 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 682698 T566 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:57673 57673
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   1> client port:0.0.0.0/0.0.0.0:57673
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CollectionsAPIDistributedZkTest -Dtests.method=testDistribSearch -Dtests.seed=D7B72BD6048F420F -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=nl -Dtests.timezone=Asia/Baku -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 97.8s J1 | CollectionsAPIDistributedZkTest.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D7B72BD6048F420F:5651A5CE73D02233]:0)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.deleteCollectionWithDownNodes(CollectionsAPIDistributedZkTest.java:390)
   [junit4]    > 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:203)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 682977 T566 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 97820 T565 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 3 leaked thread(s).
   [junit4]   2> 683925 T592 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 684120 T652 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene46: {other_tl1=PostingsFormat(name=FSTOrdPulsing41), _version_=PostingsFormat(name=FSTOrdPulsing41), multiDefault=PostingsFormat(name=Memory doPackFST= false), a_t=PostingsFormat(name=FSTOrdPulsing41), a_si=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=FSTOrdPulsing41), range_facet_sl=TestBloomFilteredLucene41Postings(BloomFilteringPostingsFormat(Lucene41(blocksize=128))), range_facet_si=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= true), text=TestBloomFilteredLucene41Postings(BloomFilteringPostingsFormat(Lucene41(blocksize=128))), range_facet_l=PostingsFormat(name=FSTOrdPulsing41), timestamp=PostingsFormat(name=FSTOrdPulsing41)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=nl, timezone=Asia/Baku
   [junit4]   2> NOTE: Linux 3.8.0-35-generic i386/Oracle Corporation 1.8.0-ea (32-bit)/cpus=8,threads=4,free=105511080,total=292552704
   [junit4]   2> NOTE: All tests run in this JVM: [TestIndexSearcher, AbstractAnalyticsStatsTest, TestSolrJ, TestCollationField, TestCodecSupport, PingRequestHandlerTest, HdfsUnloadDistributedZkTest, ZkNodePropsTest, DistanceFunctionTest, TestBinaryField, TestSweetSpotSimilarityFactory, TestNonNRTOpen, TestPhraseSuggestions, OpenExchangeRatesOrgProviderTest, TestCollapseQParserPlugin, AddBlockUpdateTest, TestNumberUtils, SolrPluginUtilsTest, SpellCheckCollatorTest, BlockDirectoryTest, SpellingQueryConverterTest, IndexBasedSpellCheckerTest, ModifyConfFileTest, AddSchemaFieldsUpdateProcessorFactoryTest, DistribCursorPagingTest, HighlighterConfigTest, SimplePostToolTest, TestCollationFieldDocValues, MultiTermTest, NoFacetTest, PreAnalyzedFieldTest, BasicDistributedZk2Test, PreAnalyzedUpdateProcessorTest, SpellCheckComponentTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed on J1 in 99.14s, 1 test, 1 failure <<< FAILURES!

[...truncated 703 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:453: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:433: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:491: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1304: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:937: There were test failures: 362 suites, 1589 tests, 1 failure, 34 ignored (6 assumptions)

Total time: 46 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.8.0-ea-b121 -client -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure