You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/01/18 22:22:15 UTC

[JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 1210 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/1210/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
	at __randomizedtesting.SeedInfo.seed([E01973DE033FE27B:61FFFDC674608247]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:172)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:131)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:126)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:772)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
	at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)




Build Log:
[...truncated 10429 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 1719213 T5915 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /od_jjq/cq
   [junit4]   2> 1719222 T5915 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1390078610952
   [junit4]   2> 1719224 T5915 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1719225 T5916 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1719327 T5915 oasc.ZkTestServer.run start zk server on port:53689
   [junit4]   2> 1719329 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1719338 T5922 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b2bf8c1 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1719338 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1719339 T5915 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1719353 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1719358 T5924 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31e8fc2a name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1719359 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1719359 T5915 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1719375 T5915 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1719384 T5915 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1719393 T5915 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1719403 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1719404 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1719421 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1719422 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1719432 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1719433 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1719444 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1719445 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1719456 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1719457 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1719469 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1719470 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1719480 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1719481 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1719492 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1719493 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1719503 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1719504 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1719513 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1719514 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1719524 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1719526 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> Using old style solr.xml
   [junit4]   2> 1719542 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1719547 T5926 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66930f41 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1719547 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1719548 T5915 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1719558 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1719562 T5928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3239c4d9 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1719562 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1719564 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 1719565 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 1719576 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 1719578 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
   [junit4]   2> 1719588 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1719589 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1719600 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 1719600 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 1719611 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 1719612 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
   [junit4]   2> 1719622 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 1719624 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
   [junit4]   2> 1719633 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 1719634 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 1719650 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1719651 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1719683 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 1719684 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 1719694 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 1719695 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 1719706 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 1719707 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
   [junit4]   2> 1720156 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1720161 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53694
   [junit4]   2> 1720162 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1720163 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1720164 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458
   [junit4]   2> 1720164 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/'
   [junit4]   2> 1720208 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/solr.xml
   [junit4]   2> 1720327 T5915 oasc.CoreContainer.<init> New CoreContainer 370136241
   [junit4]   2> 1720328 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/]
   [junit4]   2> 1720330 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1720330 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1720331 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1720332 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1720333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1720333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1720334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1720334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1720335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1720350 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1720351 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1720352 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1720352 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
   [junit4]   2> 1720353 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1720355 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1720360 T5939 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6954bbc7 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1720361 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1720370 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1720374 T5941 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f18b9ff name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1720375 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1720381 T5915 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1720394 T5915 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1720406 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1720412 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53694_od_jjq%2Fcq
   [junit4]   2> 1720416 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53694_od_jjq%2Fcq
   [junit4]   2> 1720428 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1720436 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1720451 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1720460 T5915 oasc.Overseer.start Overseer (id=91100191848005637-127.0.0.1:53694_od_jjq%2Fcq-n_0000000000) starting
   [junit4]   2> 1720471 T5915 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1720486 T5943 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1720488 T5915 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1720490 T5943 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 1720497 T5915 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1720504 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1720515 T5942 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1720523 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1720524 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1720527 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1720532 T5946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4fdc9238 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1720532 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1720536 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1720546 T5915 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1721004 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1721012 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53698
   [junit4]   2> 1721013 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1721013 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1721014 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275
   [junit4]   2> 1721014 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/'
   [junit4]   2> 1721059 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solr.xml
   [junit4]   2> 1721165 T5915 oasc.CoreContainer.<init> New CoreContainer 2108103389
   [junit4]   2> 1721165 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/]
   [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1721186 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1721187 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1721187 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1721188 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
   [junit4]   2> 1721189 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1721192 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1721207 T5957 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65d4e39d name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1721207 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1721220 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1721228 T5959 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4db4e5c6 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1721229 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1721243 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1722255 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53698_od_jjq%2Fcq
   [junit4]   2> 1722258 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53698_od_jjq%2Fcq
   [junit4]   2> 1722266 T5959 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1722266 T5941 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1722267 T5946 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1722281 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1722281 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1722625 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1722630 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53701
   [junit4]   2> 1722631 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011
   [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/'
   [junit4]   2> 1722668 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solr.xml
   [junit4]   2> 1722738 T5915 oasc.CoreContainer.<init> New CoreContainer 1501927636
   [junit4]   2> 1722739 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/]
   [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1722743 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1722743 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1722753 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1722753 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1722753 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1722754 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
   [junit4]   2> 1722754 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1722755 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1722760 T5971 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51f4b436 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1722760 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1722767 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1722771 T5973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b11c2d8 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1722771 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1722782 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1723792 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53701_od_jjq%2Fcq
   [junit4]   2> 1723795 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53701_od_jjq%2Fcq
   [junit4]   2> 1723805 T5941 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1723806 T5946 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1723807 T5959 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1723807 T5973 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1723827 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1723828 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1724210 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1724215 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53704
   [junit4]   2> 1724216 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1724216 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1724217 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557
   [junit4]   2> 1724217 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/'
   [junit4]   2> 1724246 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solr.xml
   [junit4]   2> 1724331 T5915 oasc.CoreContainer.<init> New CoreContainer 1311775510
   [junit4]   2> 1724332 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/]
   [junit4]   2> 1724333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1724333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1724346 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1724347 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1724347 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1724347 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
   [junit4]   2> 1724348 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1724349 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1724354 T5985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38426974 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1724354 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1724361 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1724364 T5987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22116710 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1724365 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1724377 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1725388 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53704_od_jjq%2Fcq
   [junit4]   2> 1725391 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53704_od_jjq%2Fcq
   [junit4]   2> 1725402 T5946 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1725402 T5941 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1725403 T5959 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1725403 T5987 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1725403 T5973 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1725420 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1725421 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1725880 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1725885 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53707
   [junit4]   2> 1725886 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1725886 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1725887 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150
   [junit4]   2> 1725888 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/'
   [junit4]   2> 1725922 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solr.xml
   [junit4]   2> 1725993 T5915 oasc.CoreContainer.<init> New CoreContainer 1201695941
   [junit4]   2> 1725994 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/]
   [junit4]   2> 1725995 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1725995 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1725997 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1725997 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1725998 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1726008 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1726008 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1726008 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1726009 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
   [junit4]   2> 1726009 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1726010 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1726014 T5999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75451e3f name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1726015 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1726022 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1726028 T6001 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@509b80a4 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1726028 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1726040 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1727054 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53707_od_jjq%2Fcq
   [junit4]   2> 1727056 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53707_od_jjq%2Fcq
   [junit4]   2> 1727068 T5959 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1727068 T5946 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1727068 T5987 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1727068 T6001 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1727068 T5941 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1727068 T5973 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1727088 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1727088 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1727135 T5953 oasha.CollectionsHandler.handleCreateAction Creating Collection : version=2&numShards=2&action=CREATE&collection.configName=conf1&router.field=myOwnField&name=solrj_collection&replicationFactor=2&wt=javabin&router.name=compositeId
   [junit4]   2> 1727138 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727142 T5943 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 1727142 T5943 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 1727143 T5943 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
   [junit4]   2> 1727143 T5943 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
   [junit4]   2> 1727156 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727160 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1727161 T5942 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
   [junit4]   2> 1727162 T5942 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
   [junit4]   2> 1727169 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727172 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727172 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727172 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727172 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727172 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727172 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727257 T5943 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 1727258 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:53698_od_jjq%2Fcq
   [junit4]   2> 1727259 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:53707_od_jjq%2Fcq
   [junit4]   2> 1727260 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:53704_od_jjq%2Fcq
   [junit4]   2> 1727262 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:53701_od_jjq%2Fcq
   [junit4]   2> 1727267 T5954 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
   [junit4]   2> 1727271 T5979 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
   [junit4]   2> 1727273 T5995 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
   [junit4]   2> 1727273 T5969 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
   [junit4]   2> 1727276 T5954 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1727277 T5979 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1727277 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727279 T5969 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1727281 T5995 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1727284 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1727287 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53698/od_jjq/cq",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53698_od_jjq%2Fcq",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1727295 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727300 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53704/od_jjq/cq",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53704_od_jjq%2Fcq",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1727310 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727314 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53701/od_jjq/cq",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53701_od_jjq%2Fcq",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1727321 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727324 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53707/od_jjq/cq",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53707_od_jjq%2Fcq",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1727332 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1727445 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727445 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727446 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727445 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727446 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1727446 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1728278 T5954 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
   [junit4]   2> 1728278 T5979 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
   [junit4]   2> 1728279 T5954 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solr.xml
   [junit4]   2> 1728279 T5979 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solr.xml
   [junit4]   2> 1728280 T5979 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solrj_collection_shard2_replica1
   [junit4]   2> 1728280 T5979 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1728281 T5969 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
   [junit4]   2> 1728280 T5954 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1
   [junit4]   2> 1728281 T5954 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1728282 T5969 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solr.xml
   [junit4]   2> 1728282 T5969 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2
   [junit4]   2> 1728282 T5969 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1728282 T5995 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
   [junit4]   2> 1728284 T5995 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solr.xml
   [junit4]   2> 1728284 T5979 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1728284 T5995 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2
   [junit4]   2> 1728284 T5954 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1728285 T5954 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1728285 T5995 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1728285 T5969 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1728286 T5969 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1728284 T5979 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1728287 T5995 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1728287 T5995 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1728289 T5954 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/'
   [junit4]   2> 1728291 T5995 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/'
   [junit4]   2> 1728292 T5969 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/'
   [junit4]   2> 1728295 T5979 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solrj_collection_shard2_replica1/'
   [junit4]   2> 1728399 T5954 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1728413 T5995 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1728426 T5969 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1728492 T5979 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1728547 T5969 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1728550 T5995 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1728554 T5969 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1728574 T5995 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1728577 T5969 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
   [junit4]   2> 1728610 T5995 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
   [junit4]   2> 1728640 T5954 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1728652 T5979 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1728653 T5954 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1728663 T5954 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
   [junit4]   2> 1728663 T5979 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1728715 T5979 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
   [junit4]   2> 1730003 T5954 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730003 T5954 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730004 T5954 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1730012 T5954 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1730018 T5954 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1730071 T5969 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730071 T5969 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730073 T5969 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1730090 T5969 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1730097 T5969 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1730128 T5995 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730128 T5995 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730129 T5995 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1730129 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730137 T5969 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730141 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730153 T5995 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1730158 T5995 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1730166 T5969 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730176 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730180 T5969 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730181 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730182 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730182 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730182 T5979 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730184 T5979 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1730184 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730184 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730185 T5969 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730186 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730186 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730187 T5954 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1730187 T5954 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/, dataDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/
   [junit4]   2> 1730188 T5954 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
   [junit4]   2> 1730185 T5979 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1730189 T5954 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data
   [junit4]   2> 1730189 T5954 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index/
   [junit4]   2> 1730190 T5954 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1730190 T5954 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index
   [junit4]   2> 1730191 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1730192 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730192 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730192 T5969 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1730193 T5969 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/, dataDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/
   [junit4]   2> 1730193 T5969 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
   [junit4]   2> 1730194 T5969 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data
   [junit4]   2> 1730195 T5969 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index/
   [junit4]   2> 1730195 T5969 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1730197 T5969 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index
   [junit4]   2> 1730197 T5969 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1730199 T5954 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@247327d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@392d8b48),segFN=segments_1,generation=1}
   [junit4]   2> 1730199 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1730199 T5969 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@434ea2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4d5719cb),segFN=segments_1,generation=1}
   [junit4]   2> 1730200 T5969 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1730200 T5979 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1730204 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1730204 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1730205 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1730206 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1730207 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1730207 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1730208 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1730209 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1730209 T5969 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1730204 T5979 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1730210 T5969 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1730212 T5969 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1730213 T5969 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1730213 T5969 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1730214 T5969 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1730215 T5969 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1730209 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1730216 T5969 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1730216 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1730217 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1730217 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1730218 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1730219 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1730220 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1730220 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1730221 T5954 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1730222 T5954 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1730222 T5954 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1730223 T5954 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1730223 T5954 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1730224 T5954 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1730227 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1730226 T5969 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1730228 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1730235 T5954 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1730240 T5969 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1730241 T5969 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1730244 T5969 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6018786168657202]
   [junit4]   2> 1730245 T5969 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@434ea2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4d5719cb),segFN=segments_1,generation=1}
   [junit4]   2> 1730245 T5969 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1730245 T5969 oass.SolrIndexSearcher.<init> Opening Searcher@682747bb main
   [junit4]   2> 1730246 T5954 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1730249 T5954 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1730249 T5995 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730251 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6018786168657202]
   [junit4]   2> 1730253 T5954 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@247327d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@392d8b48),segFN=segments_1,generation=1}
   [junit4]   2> 1730253 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1730254 T5954 oass.SolrIndexSearcher.<init> Opening Searcher@7487fd38 main
   [junit4]   2> 1730260 T6008 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@7487fd38 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1730263 T5995 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730263 T5954 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
   [junit4]   2> 1730264 T5954 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:53698/od_jjq/cq collection:solrj_collection shard:shard1
   [junit4]   2> 1730264 T6007 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica2] Registered new searcher Searcher@682747bb main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1730268 T5954 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
   [junit4]   2> 1730288 T5995 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730288 T5969 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica2
   [junit4]   2> 1730289 T5969 oasc.ZkController.register Register replica - core:solrj_collection_shard2_replica2 address:http://127.0.0.1:53701/od_jjq/cq collection:solrj_collection shard:shard2
   [junit4]   2> 1730292 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730293 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730294 T5995 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730300 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730301 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730302 T5995 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1730302 T5995 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/
   [junit4]   2> 1730303 T5995 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
   [junit4]   2> 1730294 T5969 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard2/election
   [junit4]   2> 1730308 T5995 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data
   [junit4]   2> 1730309 T5995 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index/
   [junit4]   2> 1730310 T5995 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1730311 T5995 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index
   [junit4]   2> 1730312 T5995 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1730319 T5995 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@858afaf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@659dfd2a),segFN=segments_1,generation=1}
   [junit4]   2> 1730320 T5995 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1730322 T5979 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730325 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1730331 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1730332 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1730333 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1730333 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1730334 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1730335 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1730336 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1730338 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1730339 T5995 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1730339 T5979 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1730340 T5995 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1730341 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1730343 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1730343 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1730344 T5995 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1730344 T5995 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1730346 T5995 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1730346 T5954 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=-4
   [junit4]   2> 1730349 T5954 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
   [junit4]   2> 1730349 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C4181 name=solrj_collection_shard1_replica1 org.apache.solr.core.SolrCore@46d7249d url=http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1 node=127.0.0.1:53698_od_jjq%2Fcq C4181_STATE=coll:solrj_collection core:solrj_collection_shard1_replica1 props:{state=down, base_url=http://127.0.0.1:53698/od_jjq/cq, core=solrj_collection_shard1_replica1, node_name=127.0.0.1:53698_od_jjq%2Fcq}
   [junit4]   2> 1730350 T5954 C4181 P53698 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1/
   [junit4]   2> 1730350 T5954 C4181 P53698 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:53698/od_jjq/cq START replicas=[http://127.0.0.1:53707/od_jjq/cq/solrj_collection_shard1_replica2/] nUpdates=100
   [junit4]   2> 1730353 T5995 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1730354 T5995 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1730356 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1730354 T5979 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730358 T5995 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1730361 T5969 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1730367 T5995 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1730369 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730372 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730373 T5979 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1730367 T5954 C4181 P53698 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:53698/od_jjq/cq DONE.  We have no versions.  sync failed.
   [junit4]   2> 1730374 T5954 C4181 P53698 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1730374 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1730379 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1730379 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1/ shard1
   [junit4]   2> 1730375 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1730384 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1730382 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1730381 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1730380 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1730385 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1730384 T5954 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leaders/shard1
   [junit4]   2> 1730391 T5969 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=-9
   [junit4]   2> 1730395 T5969 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
   [junit4]   2> 1730396 T5969 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1730396 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1730397 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1730397 T5979 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1730395 T5995 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1730399 T5995 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1730398 T5979 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/wo

[...truncated too long message...]

)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:273)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 	
   [junit4]   2> 2162194 T5987 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/nodes_used_collection/leader_elect/shard1/election/91100191848005644-core_node1-n_0000000001
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:173)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:170)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:170)
   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:67)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:293)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 2162195 T5987 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2162195 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 2162195 T5987 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 2162196 T5987 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> NOTE: test params are: codec=Appending, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_IQ, timezone=Etc/UTC
   [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.7.0_51 (64-bit)/cpus=2,threads=1,free=107685104,total=385253376
   [junit4]   2> NOTE: All tests run in this JVM: [PluginInfoTest, QueryElevationComponentTest, ZkCLITest, TestCollationField, CurrencyFieldXmlFileTest, BadCopyFieldTest, TestSolrXmlPersistence, TestPartialUpdateDeduplication, TestManagedSchema, WordBreakSolrSpellCheckerTest, RAMDirectoryFactoryTest, TestDistributedMissingSort, TestJmxIntegration, TestElisionMultitermQuery, SpellCheckComponentTest, ZkSolrClientTest, HdfsBasicDistributedZk2Test, TestArbitraryIndexDir, TestSimpleQParserPlugin, TestRandomDVFaceting, TestSchemaSimilarityResource, OutputWriterTest, TestQueryTypes, CopyFieldTest, TestStressReorder, DistanceFunctionTest, SuggestComponentTest, MigrateRouteKeyTest, DateFieldTest, DistributedDebugComponentTest, TestCloudManagedSchemaAddField, FieldAnalysisRequestHandlerTest, TestFileDictionaryLookup, TestDocBasedVersionConstraints, LeaderElectionIntegrationTest, TestCollapseQParserPlugin, TestMultiCoreConfBootstrap, QueryResultKeyTest, TestDocSet, TestLMDirichletSimilarityFactory, TestCSVLoader, TestRTGBase, TestSchemaVersionResource, FileUtilsTest, TriLevelCompositeIdRoutingTest, ParsingFieldUpdateProcessorsTest, TestCodecSupport, DistributedTermsComponentTest, TestCustomSort, ConnectionManagerTest, RequiredFieldsTest, TestQueryUtils, TestReplicationHandler, TestSolrXMLSerializer, TestUtils, UpdateRequestProcessorFactoryTest, TestFastWriter, TestPHPSerializedResponseWriter, OpenExchangeRatesOrgProviderTest, TestFoldingMultitermQuery, TestNonNRTOpen, DOMUtilTest, CSVRequestHandlerTest, TestIndexingPerformance, DocumentAnalysisRequestHandlerTest, TestLuceneMatchVersion, DocValuesTest, TestJoin, TestSchemaNameResource, TestBadConfig, CoreAdminCreateDiscoverTest, HighlighterConfigTest, TestPerFieldSimilarity, BlockCacheTest, SolrIndexSplitterTest, URLClassifyProcessorTest, SchemaVersionSpecificBehaviorTest, QueryParsingTest, CursorMarkTest, SimplePostToolTest, TestNumberUtils, EnumFieldTest, ChaosMonkeySafeLeaderTest, DirectUpdateHandlerOptimizeTest, TestCollationFieldDocValues, TestFuzzyAnalyzedSuggestions, TestQuerySenderListener, HdfsRecoveryZkTest, CoreMergeIndexesAdminHandlerTest, ClusterStateTest, TestSolr4Spatial, TestSystemIdResolver, TestStressLucene, SolrCoreCheckLockOnStartupTest, TestManagedSchemaFieldResource, IndexSchemaTest, TestWordDelimiterFilterFactory, TestOmitPositions, TermVectorComponentTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestIndexSearcher, TestPseudoReturnFields, HdfsChaosMonkeySafeLeaderTest, TestLazyCores, DateMathParserTest, TestAtomicUpdateErrorCases, TestLFUCache, TestDefaultSimilarityFactory, TestPostingsSolrHighlighter, SimpleFacetsTest, TestImplicitCoreProperties, DeleteReplicaTest, TestSerializedLuceneMatchVersion, TestHighFrequencyDictionaryFactory, ChangedSchemaMergeTest, TestAnalyzedSuggestions, CursorPagingTest, PingRequestHandlerTest, TestSuggestSpellingConverter, TestDFRSimilarityFactory, TestSolrDeletionPolicy2, PrimUtilsTest, SignatureUpdateProcessorFactoryTest, TestStressUserVersions, TestDistribDocBasedVersion, PrimitiveFieldTypeTest, ShardRoutingCustomTest, TestCloudManagedSchema, TestDynamicFieldCollectionResource, AutoCommitTest, SolrPluginUtilsTest, BlockDirectoryTest, SolrIndexConfigTest, ShowFileRequestHandlerTest, JSONWriterTest, TestStressVersions, SpatialFilterTest, BadIndexSchemaTest, ShardRoutingTest, TestShardHandlerFactory, SuggesterTest, TestBM25SimilarityFactory, DirectUpdateHandlerTest, DistribCursorPagingTest, FieldMutatingUpdateProcessorTest, RecoveryZkTest, TestNoOpRegenerator, TestSolrDeletionPolicy1, BasicDistributedZkTest, TestBinaryResponseWriter, TestRandomMergePolicy, ExternalFileFieldSortTest, OpenCloseCoreStressTest, PolyFieldTest, BasicDistributedZk2Test, TestFreeTextSuggestions, JsonLoaderTest, TestAddFieldRealTimeGet, UniqFieldsUpdateProcessorFactoryTest, TestAnalyzeInfixSuggestions, TestStressRecovery, TestBinaryField, IndexSchemaRuntimeFieldTest, TestSolrQueryParser, HdfsBasicDistributedZkTest, ZkNodePropsTest, BadComponentTest, UUIDFieldTest, AssignTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed in 443.02s, 1 test, 1 failure <<< FAILURES!

[...truncated 539 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:459: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:439: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/extra-targets.xml:37: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1308: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:941: There were test failures: 354 suites, 1540 tests, 1 failure, 40 ignored (6 assumptions)

Total time: 85 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 1211 - Still Failing!

Posted by Mark Miller <ma...@gmail.com>.
I filed SOLR-5645.


On Sun, Jan 19, 2014 at 12:08 PM, Mark Miller <ma...@gmail.com> wrote:

> Well, I thought it was https://issues.apache.org/jira/browse/SOLR-4805,
> but it’s not that, it’s just similar.
>
> Some clean up that still needs to be done here. It does affect 4.6.1 as
> well.
>
> - Mark
>
> On Jan 19, 2014, at 12:02 PM, Mark Miller <ma...@gmail.com> wrote:
>
> Well, this isn’t good. It’s an old bug brought back to life - its trying
> to register the core in zk again on reload - this was previously fixed.
> Hope it’s not in 4.6.1.
>
> - Mark
>
> On Jan 18, 2014, at 11:16 PM, Policeman Jenkins Server <
> jenkins@thetaphi.de> wrote:
>
> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/1211/
> Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
>
> 1 tests failed.
> FAILED:
>  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch
>
> Error Message:
> reloadcollection the collection time out:60s
>
> Stack Trace:
> org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException:
> reloadcollection the collection time out:60s
> at __randomizedtesting.SeedInfo.seed([A836D24B8F744B19:29D05C53F82B2B25]:0)
> at
> org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
> at
> org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
> at
> org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:765)
> at
> org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
> at
> org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
> at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:606)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> at
> org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> at
> com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> at java.lang.Thread.run(Thread.java:744)
>
>
>
>
> Build Log:
> [...truncated 10447 lines...]
>  [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
>  [junit4]   2> 1753662 T5108
> oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext
> system property: /
>  [junit4]   2> 1753672 T5108 oas.SolrTestCaseJ4.setUp ###Starting
> testDistribSearch
>  [junit4]   2> Creating dataDir:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1390103593043
>  [junit4]   2> 1753674 T5108 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>  [junit4]   2> 1753675 T5109 oasc.ZkTestServer$ZKServerMain.runFromConfig
> Starting server
>  [junit4]   2> 1753776 T5108 oasc.ZkTestServer.run start zk server on
> port:54105
>  [junit4]   2> 1753778 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1753786 T5115 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@692bd0f2name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1753787 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1753788 T5108 oascc.SolrZkClient.makePath makePath: /solr
>  [junit4]   2> 1753800 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1753806 T5117 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@46284bcename:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1753806 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1753807 T5108 oascc.SolrZkClient.makePath makePath:
> /collections/collection1
>  [junit4]   2> 1753816 T5108 oascc.SolrZkClient.makePath makePath:
> /collections/collection1/shards
>  [junit4]   2> 1753824 T5108 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection
>  [junit4]   2> 1753831 T5108 oascc.SolrZkClient.makePath makePath:
> /collections/control_collection/shards
>  [junit4]   2> 1753836 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml
> to /configs/conf1/solrconfig.xml
>  [junit4]   2> 1753837 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.xml
>  [junit4]   2> 1753850 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml
> to /configs/conf1/schema.xml
>  [junit4]   2> 1753851 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/schema.xml
>  [junit4]   2> 1753861 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
> to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>  [junit4]   2> 1753862 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>  [junit4]   2> 1753872 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt
> to /configs/conf1/stopwords.txt
>  [junit4]   2> 1753873 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/stopwords.txt
>  [junit4]   2> 1753886 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt
> to /configs/conf1/protwords.txt
>  [junit4]   2> 1753887 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/protwords.txt
>  [junit4]   2> 1753898 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml
> to /configs/conf1/currency.xml
>  [junit4]   2> 1753899 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/currency.xml
>  [junit4]   2> 1753909 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml
> to /configs/conf1/enumsConfig.xml
>  [junit4]   2> 1753911 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/enumsConfig.xml
>  [junit4]   2> 1753921 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json
> to /configs/conf1/open-exchange-rates.json
>  [junit4]   2> 1753922 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/open-exchange-rates.json
>  [junit4]   2> 1753930 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
> to /configs/conf1/mapping-ISOLatin1Accent.txt
>  [junit4]   2> 1753931 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/mapping-ISOLatin1Accent.txt
>  [junit4]   2> 1753941 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt
> to /configs/conf1/old_synonyms.txt
>  [junit4]   2> 1753942 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/old_synonyms.txt
>  [junit4]   2> 1753953 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt
> to /configs/conf1/synonyms.txt
>  [junit4]   2> 1753954 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf1/synonyms.txt
>  [junit4]   2> Using old style solr.xml
>  [junit4]   2> 1753972 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1753978 T5119 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@218b21e5name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1753978 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1753979 T5108 oascc.SolrZkClient.makePath makePath: /solr
>  [junit4]   2> 1753986 T5110 oazs.NIOServerCnxn.doIO WARN caught end of
> stream exception EndOfStreamException: Unable to read additional data from
> client sessionid 0x143a8a1805b0002, likely client has closed socket
>  [junit4]   2> at
> org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
>  [junit4]   2> at
> org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>  [junit4]   2> at java.lang.Thread.run(Thread.java:744)
>  [junit4]   2>
>  [junit4]   2> 1753990 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1753999 T5121 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@3cb1f3aname:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1754000 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1754003 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml
> to /configs/conf2/solrconfig.xml
>  [junit4]   2> 1754005 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/solrconfig.xml
>  [junit4]   2> 1754018 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml
> to /configs/conf2/schema.xml
>  [junit4]   2> 1754021 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/schema.xml
>  [junit4]   2> 1754036 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml
> to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>  [junit4]   2> 1754037 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>  [junit4]   2> 1754045 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt
> to /configs/conf2/stopwords.txt
>  [junit4]   2> 1754046 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/stopwords.txt
>  [junit4]   2> 1754061 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt
> to /configs/conf2/protwords.txt
>  [junit4]   2> 1754063 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/protwords.txt
>  [junit4]   2> 1754071 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml
> to /configs/conf2/currency.xml
>  [junit4]   2> 1754073 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/currency.xml
>  [junit4]   2> 1754086 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json
> to /configs/conf2/open-exchange-rates.json
>  [junit4]   2> 1754087 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/open-exchange-rates.json
>  [junit4]   2> 1754105 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt
> to /configs/conf2/mapping-ISOLatin1Accent.txt
>  [junit4]   2> 1754106 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/mapping-ISOLatin1Accent.txt
>  [junit4]   2> 1754115 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt
> to /configs/conf2/old_synonyms.txt
>  [junit4]   2> 1754116 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/old_synonyms.txt
>  [junit4]   2> 1754128 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt
> to /configs/conf2/synonyms.txt
>  [junit4]   2> 1754129 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/synonyms.txt
>  [junit4]   2> 1754139 T5108 oasc.AbstractZkTestCase.putConfig put
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml
> to /configs/conf2/elevate.xml
>  [junit4]   2> 1754140 T5108 oascc.SolrZkClient.makePath makePath:
> /configs/conf2/elevate.xml
>  [junit4]   2> 1754561 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>  [junit4]   2> 1754575 T5108 oejs.AbstractConnector.doStart Started
> SelectChannelConnector@127.0.0.1:54110
>  [junit4]   2> 1754576 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>  [junit4]   2> 1754576 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>  [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521
>  [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/'
>  [junit4]   2> 1754614 T5108 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
>  [junit4]   2> 1754697 T5108 oasc.CoreContainer.<init> New CoreContainer
> 725528504
>  [junit4]   2> 1754697 T5108 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/]
>  [junit4]   2> 1754699 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 120000
>  [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to: http://
>  [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>  [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>  [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>  [junit4]   2> 1754713 T5108 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>  [junit4]   2> 1754714 T5108 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>  [junit4]   2> 1754714 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>  [junit4]   2> 1754715 T5108 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:54105/solr
>  [junit4]   2> 1754715 T5108 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>  [junit4]   2> 1754716 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1754721 T5132 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@20826f36name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1754722 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1754728 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1754733 T5134 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@3ee56b7aname:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1754733 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1754737 T5108 oascc.SolrZkClient.makePath makePath:
> /overseer/queue
>  [junit4]   2> 1754748 T5108 oascc.SolrZkClient.makePath makePath:
> /overseer/collection-queue-work
>  [junit4]   2> 1754758 T5108 oascc.SolrZkClient.makePath makePath:
> /live_nodes
>  [junit4]   2> 1754764 T5108 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54110_
>  [junit4]   2> 1754766 T5108 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:54110_
>  [junit4]   2> 1754776 T5108 oascc.SolrZkClient.makePath makePath:
> /overseer_elect
>  [junit4]   2> 1754783 T5108 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/election
>  [junit4]   2> 1754796 T5108 oascc.SolrZkClient.makePath makePath:
> /overseer_elect/leader
>  [junit4]   2> 1754802 T5108 oasc.Overseer.start Overseer
> (id=91101829074190341-127.0.0.1:54110_-n_0000000000) starting
>  [junit4]   2> 1754813 T5108 oascc.SolrZkClient.makePath makePath:
> /overseer/queue-work
>  [junit4]   2> 1754826 T5136 oasc.OverseerCollectionProcessor.run Process
> current queue of collection creations
>  [junit4]   2> 1754827 T5108 oascc.SolrZkClient.makePath makePath:
> /clusterstate.json
>  [junit4]   2> 1754829 T5136
> oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing
> overseer nodes
>  [junit4]   2> 1754835 T5108 oascc.SolrZkClient.makePath makePath:
> /aliases.json
>  [junit4]   2> 1754842 T5108
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>  [junit4]   2> 1754851 T5135 oasc.Overseer$ClusterStateUpdater.run
> Starting to work on the main queue
>  [junit4]   2> 1754860 T5108 oass.SolrDispatchFilter.init
> user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>  [junit4]   2> 1754861 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>  [junit4]   2> 1754862 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1754867 T5139 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@5bbaf1afname:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1754868 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1754871 T5108
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>  [junit4]   2> 1754879 T5108 oasc.ChaosMonkey.monkeyLog monkey: init -
> expire sessions:false cause connection loss:false
>  [junit4]   2> 1755261 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>  [junit4]   2> 1755265 T5108 oejs.AbstractConnector.doStart Started
> SelectChannelConnector@127.0.0.1:54114
>  [junit4]   2> 1755268 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>  [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>  [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250
>  [junit4]   2> 1755269 T5108 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/'
>  [junit4]   2> 1755308 T5108 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
>  [junit4]   2> 1755412 T5108 oasc.CoreContainer.<init> New CoreContainer
> 96924362
>  [junit4]   2> 1755412 T5108 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/]
>  [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 120000
>  [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to: http://
>  [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>  [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>  [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>  [junit4]   2> 1755429 T5108 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>  [junit4]   2> 1755429 T5108 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>  [junit4]   2> 1755430 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>  [junit4]   2> 1755430 T5108 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:54105/solr
>  [junit4]   2> 1755430 T5108 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>  [junit4]   2> 1755433 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1755438 T5150 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@486daebcname:ZooKeeperConnection Watcher:
> 127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1755439 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1755446 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1755451 T5152 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@4d708b92name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1755452 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1755471 T5108
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>  [junit4]   2> 1756488 T5108 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54114_
>  [junit4]   2> 1756493 T5108 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:54114_
>  [junit4]   2> 1756508 T5134 oascc.ZkStateReader$3.process Updating live
> nodes... (2)
>  [junit4]   2> 1756508 T5139 oascc.ZkStateReader$3.process Updating live
> nodes... (2)
>  [junit4]   2> 1756510 T5152 oascc.ZkStateReader$3.process Updating live
> nodes... (2)
>  [junit4]   2> 1756538 T5108 oass.SolrDispatchFilter.init
> user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>  [junit4]   2> 1756539 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>  [junit4]   2> 1756946 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>  [junit4]   2> 1756951 T5108 oejs.AbstractConnector.doStart Started
> SelectChannelConnector@127.0.0.1:54117
>  [junit4]   2> 1756954 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>  [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>  [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911
>  [junit4]   2> 1756955 T5108 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/'
>  [junit4]   2> 1756992 T5108 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/solr.xml
>  [junit4]   2> 1757060 T5108 oasc.CoreContainer.<init> New CoreContainer
> 1734112660
>  [junit4]   2> 1757061 T5108 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/]
>  [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 120000
>  [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to: http://
>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>  [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>  [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>  [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>  [junit4]   2> 1757074 T5108 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>  [junit4]   2> 1757074 T5108 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>  [junit4]   2> 1757075 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>  [junit4]   2> 1757075 T5108 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:54105/solr
>  [junit4]   2> 1757075 T5108 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>  [junit4]   2> 1757077 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1757085 T5164 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@672b7c8fname:ZooKeeperConnection Watcher:
> 127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1757086 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1757094 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1757099 T5166 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@81214c4name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1757100 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1757110 T5108
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>  [junit4]   2> 1758123 T5108 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54117_
>  [junit4]   2> 1758128 T5108 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:54117_
>  [junit4]   2> 1758152 T5152 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>  [junit4]   2> 1758155 T5139 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>  [junit4]   2> 1758154 T5134 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>  [junit4]   2> 1758155 T5166 oascc.ZkStateReader$3.process Updating live
> nodes... (3)
>  [junit4]   2> 1758174 T5108 oass.SolrDispatchFilter.init
> user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>  [junit4]   2> 1758175 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>  [junit4]   2> 1758552 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>  [junit4]   2> 1758557 T5108 oejs.AbstractConnector.doStart Started
> SelectChannelConnector@127.0.0.1:54120
>  [junit4]   2> 1758559 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>  [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>  [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547
>  [junit4]   2> 1758561 T5108 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/'
>  [junit4]   2> 1758598 T5108 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
>  [junit4]   2> 1758704 T5108 oasc.CoreContainer.<init> New CoreContainer
> 76332827
>  [junit4]   2> 1758705 T5108 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/]
>  [junit4]   2> 1758706 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 120000
>  [junit4]   2> 1758708 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to: http://
>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>  [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>  [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>  [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>  [junit4]   2> 1758721 T5108 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>  [junit4]   2> 1758721 T5108 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>  [junit4]   2> 1758722 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>  [junit4]   2> 1758722 T5108 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:54105/solr
>  [junit4]   2> 1758722 T5108 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>  [junit4]   2> 1758724 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1758728 T5178 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@345687f2name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1758730 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1758737 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1758743 T5180 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@fde9546name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1758744 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1758758 T5108
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>  [junit4]   2> 1759770 T5108 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54120_
>  [junit4]   2> 1759776 T5108 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:54120_
>  [junit4]   2> 1759791 T5166 oascc.ZkStateReader$3.process Updating live
> nodes... (4)
>  [junit4]   2> 1759792 T5139 oascc.ZkStateReader$3.process Updating live
> nodes... (4)
>  [junit4]   2> 1759800 T5180 oascc.ZkStateReader$3.process Updating live
> nodes... (4)
>  [junit4]   2> 1759801 T5152 oascc.ZkStateReader$3.process Updating live
> nodes... (4)
>  [junit4]   2> 1759802 T5134 oascc.ZkStateReader$3.process Updating live
> nodes... (4)
>  [junit4]   2> 1759829 T5108 oass.SolrDispatchFilter.init
> user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>  [junit4]   2> 1759830 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>  [junit4]   2> 1760202 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>  [junit4]   2> 1760208 T5108 oejs.AbstractConnector.doStart Started
> SelectChannelConnector@127.0.0.1:54123
>  [junit4]   2> 1760210 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init()
>  [junit4]   2> 1760211 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI
> not configured for solr (NoInitialContextEx)
>  [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.locateSolrHome using
> system property solr.solr.home:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201
>  [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/'
>  [junit4]   2> 1760251 T5108 oasc.ConfigSolr.fromFile Loading container
> configuration from
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
>  [junit4]   2> 1760338 T5108 oasc.CoreContainer.<init> New CoreContainer
> 723834019
>  [junit4]   2> 1760339 T5108 oasc.CoreContainer.load Loading cores into
> CoreContainer
> [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/]
>  [junit4]   2> 1760340 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting socketTimeout to: 120000
>  [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting urlScheme to: http://
>  [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting connTimeout to: 15000
>  [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxConnectionsPerHost to: 20
>  [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting corePoolSize to: 0
>  [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maximumPoolSize to: 2147483647
>  [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting maxThreadIdleTime to: 5
>  [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting sizeOfQueue to: -1
>  [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter
> Setting fairnessPolicy to: false
>  [junit4]   2> 1760353 T5108 oasl.LogWatcher.createWatcher SLF4J impl is
> org.slf4j.impl.Log4jLoggerFactory
>  [junit4]   2> 1760354 T5108 oasl.LogWatcher.newRegisteredLogWatcher
> Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>  [junit4]   2> 1760354 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>  [junit4]   2> 1760355 T5108 oasc.ZkContainer.initZooKeeper Zookeeper
> client=127.0.0.1:54105/solr
>  [junit4]   2> 1760355 T5108 oasc.ZkController.checkChrootPath zkHost
> includes chroot
>  [junit4]   2> 1760357 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1760361 T5192 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@7f042f75name:ZooKeeperConnection Watcher:
> 127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1760363 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1760370 T5108 oascc.ConnectionManager.waitForConnected
> Waiting for client to connect to ZooKeeper
>  [junit4]   2> 1760375 T5194 oascc.ConnectionManager.process Watcher
> org.apache.solr.common.cloud.ConnectionManager@3326075aname:ZooKeeperConnection Watcher:
> 127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None
> path:null path:null type:None
>  [junit4]   2> 1760375 T5108 oascc.ConnectionManager.waitForConnected
> Client is connected to ZooKeeper
>  [junit4]   2> 1760387 T5108
> oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster
> state from ZooKeeper...
>  [junit4]   2> 1761398 T5108 oasc.ZkController.createEphemeralLiveNode
> Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54123_
>  [junit4]   2> 1761407 T5108 oascc.SolrZkClient.makePath makePath:
> /live_nodes/127.0.0.1:54123_
>  [junit4]   2> 1761415 T5180 oascc.ZkStateReader$3.process Updating live
> nodes... (5)
>  [junit4]   2> 1761416 T5134 oascc.ZkStateReader$3.process Updating live
> nodes... (5)
>  [junit4]   2> 1761417 T5194 oascc.ZkStateReader$3.process Updating live
> nodes... (5)
>  [junit4]   2> 1761417 T5152 oascc.ZkStateReader$3.process Updating live
> nodes... (5)
>  [junit4]   2> 1761416 T5139 oascc.ZkStateReader$3.process Updating live
> nodes... (5)
>  [junit4]   2> 1761417 T5166 oascc.ZkStateReader$3.process Updating live
> nodes... (5)
>  [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init
> user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>  [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init
> SolrDispatchFilter.init() done
>  [junit4]   2> 1762088 T5147 oasha.CollectionsHandler.handleCreateAction
> Creating Collection :
> router.field=myOwnField&version=2&name=solrj_collection&action=CREATE&wt=javabin&
> router.name
> =compositeId&numShards=2&collection.configName=conf1&replicationFactor=2
>  [junit4]   2> 1762091 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/collection-queue-work state: SyncConnected type
> NodeChildrenChanged
>  [junit4]   2> 1762094 T5136 oasc.OverseerCollectionProcessor.run Overseer
> Collection Processor: Get the message
> id:/overseer/collection-queue-work/qn-0000000000 message:{
>  [junit4]   2>   "operation":"createcollection",
>  [junit4]   2>   "fromApi":"true",
>  [junit4]   2>   "name":"solrj_collection",
>  [junit4]   2>   "replicationFactor":"2",
>  [junit4]   2>   "collection.configName":"conf1",
>  [junit4]   2>   "numShards":"2",
>  [junit4]   2>   "router.field":"myOwnField",
>  [junit4]   2>   "router.name":"compositeId"}
>  [junit4]   2> 1762094 T5136
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>  [junit4]   2>   "operation":"createcollection",
>  [junit4]   2>   "fromApi":"true",
>  [junit4]   2>   "name":"solrj_collection",
>  [junit4]   2>   "replicationFactor":"2",
>  [junit4]   2>   "collection.configName":"conf1",
>  [junit4]   2>   "numShards":"2",
>  [junit4]   2>   "router.field":"myOwnField",
>  [junit4]   2>   "router.name":"compositeId"}
>  [junit4]   2> 1762095 T5136
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/solrj_collection
>  [junit4]   2> 1762095 T5136 oascc.SolrZkClient.makePath makePath:
> /collections/solrj_collection
>  [junit4]   2> 1762107 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1762112 T5135 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>  [junit4]   2> 1762113 T5135
> oasc.Overseer$ClusterStateUpdater.buildCollection building a new
> collection: solrj_collection
>  [junit4]   2> 1762114 T5135
> oasc.Overseer$ClusterStateUpdater.createCollection Create collection
> solrj_collection with shards [shard1, shard2]
>  [junit4]   2> 1762121 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1762123 T5152 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762123 T5139 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762124 T5194 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762123 T5180 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762124 T5166 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762124 T5134 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762208 T5136
> oasc.OverseerCollectionProcessor.createCollection going to create cores
> replicas shardNames [shard1, shard2] , repFactor : 2
>  [junit4]   2> 1762209 T5136
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> solrj_collection_shard1_replica1 as part of slice shard1 of collection
> solrj_collection on 127.0.0.1:54114_
>  [junit4]   2> 1762209 T5136
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> solrj_collection_shard1_replica2 as part of slice shard1 of collection
> solrj_collection on 127.0.0.1:54120_
>  [junit4]   2> 1762210 T5136
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> solrj_collection_shard2_replica1 as part of slice shard2 of collection
> solrj_collection on 127.0.0.1:54110_
>  [junit4]   2> 1762210 T5136
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> solrj_collection_shard2_replica2 as part of slice shard2 of collection
> solrj_collection on 127.0.0.1:54123_
>  [junit4]   2> 1762222 T5148 oasc.ZkController.publish publishing
> core=solrj_collection_shard1_replica1 state=down
>  [junit4]   2> 1762223 T5126 oasc.ZkController.publish publishing
> core=solrj_collection_shard2_replica1 state=down
>  [junit4]   2> 1762224 T5174 oasc.ZkController.publish publishing
> core=solrj_collection_shard1_replica2 state=down
>  [junit4]   2> 1762224 T5190 oasc.ZkController.publish publishing
> core=solrj_collection_shard2_replica2 state=down
>  [junit4]   2> 1762227 T5148 oasc.ZkController.waitForCoreNodeName look
> for our core node name
>  [junit4]   2> 1762228 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1762228 T5126 oasc.ZkController.waitForCoreNodeName look
> for our core node name
>  [junit4]   2> 1762229 T5174 oasc.ZkController.waitForCoreNodeName look
> for our core node name
>  [junit4]   2> 1762229 T5190 oasc.ZkController.waitForCoreNodeName look
> for our core node name
>  [junit4]   2> 1762233 T5135 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>  [junit4]   2> 1762236 T5135 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=2 message={
>  [junit4]   2>   "operation":"state",
>  [junit4]   2>   "state":"down",
>  [junit4]   2>   "base_url":"http://127.0.0.1:54114",
>  [junit4]   2>   "core":"solrj_collection_shard1_replica1",
>  [junit4]   2>   "roles":null,
>  [junit4]   2>   "node_name":"127.0.0.1:54114_",
>  [junit4]   2>   "shard":"shard1",
>  [junit4]   2>   "shard_range":null,
>  [junit4]   2>   "shard_state":"active",
>  [junit4]   2>   "shard_parent":null,
>  [junit4]   2>   "collection":"solrj_collection",
>  [junit4]   2>   "numShards":"2",
>  [junit4]   2>   "core_node_name":null}
>  [junit4]   2> 1762243 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1762246 T5135 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=2 message={
>  [junit4]   2>   "operation":"state",
>  [junit4]   2>   "state":"down",
>  [junit4]   2>   "base_url":"http://127.0.0.1:54110",
>  [junit4]   2>   "core":"solrj_collection_shard2_replica1",
>  [junit4]   2>   "roles":null,
>  [junit4]   2>   "node_name":"127.0.0.1:54110_",
>  [junit4]   2>   "shard":"shard2",
>  [junit4]   2>   "shard_range":null,
>  [junit4]   2>   "shard_state":"active",
>  [junit4]   2>   "shard_parent":null,
>  [junit4]   2>   "collection":"solrj_collection",
>  [junit4]   2>   "numShards":"2",
>  [junit4]   2>   "core_node_name":null}
>  [junit4]   2> 1762254 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1762257 T5135 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=2 message={
>  [junit4]   2>   "operation":"state",
>  [junit4]   2>   "state":"down",
>  [junit4]   2>   "base_url":"http://127.0.0.1:54123",
>  [junit4]   2>   "core":"solrj_collection_shard2_replica2",
>  [junit4]   2>   "roles":null,
>  [junit4]   2>   "node_name":"127.0.0.1:54123_",
>  [junit4]   2>   "shard":"shard2",
>  [junit4]   2>   "shard_range":null,
>  [junit4]   2>   "shard_state":"active",
>  [junit4]   2>   "shard_parent":null,
>  [junit4]   2>   "collection":"solrj_collection",
>  [junit4]   2>   "numShards":"2",
>  [junit4]   2>   "core_node_name":null}
>  [junit4]   2> 1762265 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1762267 T5135 oasc.Overseer$ClusterStateUpdater.updateState
> Update state numShards=2 message={
>  [junit4]   2>   "operation":"state",
>  [junit4]   2>   "state":"down",
>  [junit4]   2>   "base_url":"http://127.0.0.1:54120",
>  [junit4]   2>   "core":"solrj_collection_shard1_replica2",
>  [junit4]   2>   "roles":null,
>  [junit4]   2>   "node_name":"127.0.0.1:54120_",
>  [junit4]   2>   "shard":"shard1",
>  [junit4]   2>   "shard_range":null,
>  [junit4]   2>   "shard_state":"active",
>  [junit4]   2>   "shard_parent":null,
>  [junit4]   2>   "collection":"solrj_collection",
>  [junit4]   2>   "numShards":"2",
>  [junit4]   2>   "core_node_name":null}
>  [junit4]   2> 1762276 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1762383 T5152 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762383 T5166 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762383 T5134 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762383 T5139 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762383 T5194 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1762383 T5180 oascc.ZkStateReader$2.process A cluster state
> change: WatchedEvent state:SyncConnected type:NodeDataChanged
> path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>  [junit4]   2> 1763229 T5148 oasc.ZkController.waitForShardId waiting to
> find shard id in clusterstate for solrj_collection_shard1_replica1
>  [junit4]   2> 1763230 T5148 oasc.SolrXMLCoresLocator.doPersist Persisted
> core descriptions to
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
>  [junit4]   2> 1763231 T5148 oasc.CoreContainer.create Creating SolrCore
> 'solrj_collection_shard1_replica1' using instanceDir:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1
>  [junit4]   2> 1763231 T5148 oasc.ZkController.createCollectionZkNode
> Check for collection zkNode:solrj_collection
>  [junit4]   2> 1763230 T5126 oasc.ZkController.waitForShardId waiting to
> find shard id in clusterstate for solrj_collection_shard2_replica1
>  [junit4]   2> 1763233 T5148 oasc.ZkController.createCollectionZkNode
> Collection zkNode exists
>  [junit4]   2> 1763233 T5126 oasc.SolrXMLCoresLocator.doPersist Persisted
> core descriptions to
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
>  [junit4]   2> 1763234 T5126 oasc.CoreContainer.create Creating SolrCore
> 'solrj_collection_shard2_replica1' using instanceDir:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1
>  [junit4]   2> 1763234 T5126 oasc.ZkController.createCollectionZkNode
> Check for collection zkNode:solrj_collection
>  [junit4]   2> 1763231 T5190 oasc.ZkController.waitForShardId waiting to
> find shard id in clusterstate for solrj_collection_shard2_replica2
>  [junit4]   2> 1763231 T5174 oasc.ZkController.waitForShardId waiting to
> find shard id in clusterstate for solrj_collection_shard1_replica2
>  [junit4]   2> 1763236 T5126 oasc.ZkController.createCollectionZkNode
> Collection zkNode exists
>  [junit4]   2> 1763237 T5126 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/solrj_collection
>  [junit4]   2> 1763236 T5190 oasc.SolrXMLCoresLocator.doPersist Persisted
> core descriptions to
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
>  [junit4]   2> 1763238 T5190 oasc.CoreContainer.create Creating SolrCore
> 'solrj_collection_shard2_replica2' using instanceDir:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2
>  [junit4]   2> 1763239 T5190 oasc.ZkController.createCollectionZkNode
> Check for collection zkNode:solrj_collection
>  [junit4]   2> 1763233 T5148 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/solrj_collection
>  [junit4]   2> 1763240 T5126 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/'
>  [junit4]   2> 1763237 T5174 oasc.SolrXMLCoresLocator.doPersist Persisted
> core descriptions to
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
>  [junit4]   2> 1763243 T5174 oasc.CoreContainer.create Creating SolrCore
> 'solrj_collection_shard1_replica2' using instanceDir:
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2
>  [junit4]   2> 1763243 T5174 oasc.ZkController.createCollectionZkNode
> Check for collection zkNode:solrj_collection
>  [junit4]   2> 1763242 T5190 oasc.ZkController.createCollectionZkNode
> Collection zkNode exists
>  [junit4]   2> 1763245 T5190 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/solrj_collection
>  [junit4]   2> 1763247 T5174 oasc.ZkController.createCollectionZkNode
> Collection zkNode exists
>  [junit4]   2> 1763250 T5174 oascc.ZkStateReader.readConfigName Load
> collection config from:/collections/solrj_collection
>  [junit4]   2> 1763249 T5148 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/'
>  [junit4]   2> 1763269 T5190 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/'
>  [junit4]   2> 1763277 T5174 oasc.SolrResourceLoader.<init> new
> SolrResourceLoader for directory:
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/'
>  [junit4]   2> 1763420 T5148 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_47
>  [junit4]   2> 1763477 T5126 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_47
>  [junit4]   2> 1763482 T5190 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_47
>  [junit4]   2> 1763477 T5174 oasc.SolrConfig.<init> Using Lucene
> MatchVersion: LUCENE_47
>  [junit4]   2> 1763684 T5174 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>  [junit4]   2> 1763688 T5174 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>  [junit4]   2> 1763689 T5126 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>  [junit4]   2> 1763703 T5174 oass.IndexSchema.readSchema
> [solrj_collection_shard1_replica2] Schema name=test
>  [junit4]   2> 1763718 T5126 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>  [junit4]   2> 1763735 T5126 oass.IndexSchema.readSchema
> [solrj_collection_shard2_replica1] Schema name=test
>  [junit4]   2> 1763767 T5148 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>  [junit4]   2> 1763798 T5190 oasc.SolrConfig.<init> Loaded SolrConfig:
> solrconfig.xml
>  [junit4]   2> 1763809 T5148 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>  [junit4]   2> 1763813 T5190 oass.IndexSchema.readSchema Reading Solr
> Schema from schema.xml
>  [junit4]   2> 1763838 T5190 oass.IndexSchema.readSchema
> [solrj_collection_shard2_replica2] Schema name=test
>  [junit4]   2> 1763843 T5148 oass.IndexSchema.readSchema
> [solrj_collection_shard1_replica1] Schema name=test
>  [junit4]   2> 1765449 T5126 oass.ByteField.init WARN ByteField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765457 T5126 oass.ShortField.init WARN ShortField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765458 T5126 oass.OpenExchangeRatesOrgProvider.init
> Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>  [junit4]   2> 1765534 T5126 oass.IndexSchema.readSchema default search
> field in schema is text
>  [junit4]   2> 1765545 T5126 oass.IndexSchema.readSchema unique key field:
> id
>  [junit4]   2> 1765568 T5148 oass.ByteField.init WARN ByteField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765568 T5148 oass.ShortField.init WARN ShortField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765599 T5174 oass.ByteField.init WARN ByteField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765604 T5174 oass.ShortField.init WARN ShortField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765605 T5174 oass.OpenExchangeRatesOrgProvider.init
> Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>  [junit4]   2> 1765612 T5174 oass.IndexSchema.readSchema default search
> field in schema is text
>  [junit4]   2> 1765601 T5148 oass.OpenExchangeRatesOrgProvider.init
> Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>  [junit4]   2> 1765635 T5174 oass.IndexSchema.readSchema unique key field:
> id
>  [junit4]   2> 1765649 T5148 oass.IndexSchema.readSchema default search
> field in schema is text
>  [junit4]   2> 1765665 T5148 oass.IndexSchema.readSchema unique key field:
> id
>  [junit4]   2> 1765675 T5126 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765685 T5126 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765693 T5126 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765698 T5126
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765699 T5126
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765699 T5126 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765704 T5126
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765705 T5126
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765705 T5126 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>  [junit4]   2> 1765706 T5126 oasc.SolrCore.<init>
> [solrj_collection_shard2_replica1] Opening new SolrCore at
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/,
> dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/
>  [junit4]   2> 1765707 T5126 oasc.JmxMonitoredMap.<init> JMX monitoring is
> enabled. Adding Solr mbeans to JMX Server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>  [junit4]   2> 1765706 T5190 oass.ByteField.init WARN ByteField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765712 T5174 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765713 T5190 oass.ShortField.init WARN ShortField is
> deprecated and will be removed in 5.0. You should use TrieIntField instead.
>  [junit4]   2> 1765711 T5126 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data
>  [junit4]   2> 1765721 T5174 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765718 T5190 oass.OpenExchangeRatesOrgProvider.init
> Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>  [junit4]   2> 1765730 T5190 oass.IndexSchema.readSchema default search
> field in schema is text
>  [junit4]   2> 1765735 T5174 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765722 T5126 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index/
>  [junit4]   2> 1765739 T5126 oasc.SolrCore.initIndex WARN
> [solrj_collection_shard2_replica1] Solr index directory
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index'
> doesn't exist. Creating new index...
>  [junit4]   2> 1765739 T5126 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index
>  [junit4]   2> 1765740 T5126 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>  [junit4]   2> 1765740 T5190 oass.IndexSchema.readSchema unique key field:
> id
>  [junit4]   2> 1765743 T5174
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765746 T5174
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765749 T5174 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765745 T5148 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765756 T5174
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765757 T5174
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765758 T5174 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>  [junit4]   2> 1765758 T5174 oasc.SolrCore.<init>
> [solrj_collection_shard1_replica2] Opening new SolrCore at
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/,
> dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/
>  [junit4]   2> 1765758 T5174 oasc.JmxMonitoredMap.<init> JMX monitoring is
> enabled. Adding Solr mbeans to JMX Server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>  [junit4]   2> 1765748 T5126 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>  [junit4]   2>
> commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6
> ),segFN=segments_1,generation=1}
>  [junit4]   2> 1765760 T5126 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>  [junit4]   2> 1765761 T5174 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data
>  [junit4]   2> 1765768 T5126 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "nodistrib"
>  [junit4]   2> 1765768 T5148 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765769 T5174 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index/
>  [junit4]   2> 1765770 T5174 oasc.SolrCore.initIndex WARN
> [solrj_collection_shard1_replica2] Solr index directory
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index'
> doesn't exist. Creating new index...
>  [junit4]   2> 1765771 T5174 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index
>  [junit4]   2> 1765773 T5174 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>  [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>  [junit4]   2>
> commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede
> ),segFN=segments_1,generation=1}
>  [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>  [junit4]   2> 1765778 T5148 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765769 T5126 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765780 T5174 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "nodistrib"
>  [junit4]   2> 1765781 T5174 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765783 T5126 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765784 T5174 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>  [junit4]   2> 1765782 T5148
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765785 T5174 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765786 T5174 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765786 T5174 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>  [junit4]   2> 1765788 T5174 oasc.RequestHandlers.initHandlersFromConfig
> created standard: solr.StandardRequestHandler
>  [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig
> created /get: solr.RealTimeGetHandler
>  [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig
> adding lazy requestHandler: solr.ReplicationHandler
>  [junit4]   2> 1765791 T5174 oasc.RequestHandlers.initHandlersFromConfig
> created /replication: solr.ReplicationHandler
>  [junit4]   2> 1765785 T5148
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765792 T5148 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765784 T5126 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>  [junit4]   2> 1765792 T5174 oasc.RequestHandlers.initHandlersFromConfig
> created /update: solr.UpdateRequestHandler
>  [junit4]   2> 1765795 T5126 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765795 T5174 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/: org.apache.solr.handler.admin.AdminHandlers
>  [junit4]   2> 1765797 T5148
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765799 T5148
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765799 T5148 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>  [junit4]   2> 1765797 T5126 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765800 T5126 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>  [junit4]   2> 1765800 T5148 oasc.SolrCore.<init>
> [solrj_collection_shard1_replica1] Opening new SolrCore at
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/,
> dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/
>  [junit4]   2> 1765802 T5126 oasc.RequestHandlers.initHandlersFromConfig
> created standard: solr.StandardRequestHandler
>  [junit4]   2> 1765802 T5190 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765798 T5174 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/file: solr.admin.ShowFileRequestHandler
>  [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig
> created /get: solr.RealTimeGetHandler
>  [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig
> adding lazy requestHandler: solr.ReplicationHandler
>  [junit4]   2> 1765804 T5126 oasc.RequestHandlers.initHandlersFromConfig
> created /replication: solr.ReplicationHandler
>  [junit4]   2> 1765802 T5148 oasc.JmxMonitoredMap.<init> JMX monitoring is
> enabled. Adding Solr mbeans to JMX Server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>  [junit4]   2> 1765807 T5126 oasc.RequestHandlers.initHandlersFromConfig
> created /update: solr.UpdateRequestHandler
>  [junit4]   2> 1765809 T5126 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/: org.apache.solr.handler.admin.AdminHandlers
>  [junit4]   2> 1765810 T5126 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/file: solr.admin.ShowFileRequestHandler
>  [junit4]   2> 1765813 T5148 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data
>  [junit4]   2> 1765814 T5148 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index/
>  [junit4]   2> 1765814 T5148 oasc.SolrCore.initIndex WARN
> [solrj_collection_shard1_replica1] Solr index directory
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index'
> doesn't exist. Creating new index...
>  [junit4]   2> 1765816 T5126 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>  [junit4]   2> 1765820 T5148 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index
>  [junit4]   2> 1765820 T5148 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>  [junit4]   2> 1765817 T5174 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>  [junit4]   2> 1765823 T5190 oass.FileExchangeRateProvider.reload
> Reloading exchange rates from file currency.xml
>  [junit4]   2> 1765823 T5126 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>  [junit4]   2> 1765825 T5126 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>  [junit4]   2> 1765826 T5126 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy:
> minMergeSize=0, mergeFactor=10, maxMergeSize=892100269,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>  [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>  [junit4]   2>
> commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6
> ),segFN=segments_1,generation=1}
>  [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>  [junit4]   2> 1765828 T5126 oass.SolrIndexSearcher.<init> Opening
> Searcher@190a2642 main
>  [junit4]   2> 1765829 T5148 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>  [junit4]   2>
> commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414
> ),segFN=segments_1,generation=1}
>  [junit4]   2> 1765830 T5148 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>  [junit4]   2> 1765834 T5200 oasc.SolrCore.registerSearcher
> [solrj_collection_shard2_replica1] Registered new searcher Searcher@190a2642main{StandardDirectoryReader(segments_1:1:nrt)}
>  [junit4]   2> 1765837 T5126 oasc.CoreContainer.registerCore registering
> core: solrj_collection_shard2_replica1
>  [junit4]   2> 1765837 T5126 oasc.ZkController.register Register replica -
> core:solrj_collection_shard2_replica1 address:http://127.0.0.1:54110collection:solrj_collection shard:shard2
>  [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "nodistrib"
>  [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765842 T5148 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>  [junit4]   2> 1765844 T5148 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765844 T5126 oascc.SolrZkClient.makePath makePath:
> /collections/solrj_collection/leader_elect/shard2/election
>  [junit4]   2> 1765841 T5190 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765845 T5148 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765846 T5148 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>  [junit4]   2> 1765842 T5174 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>  [junit4]   2> 1765848 T5174 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>  [junit4]   2> 1765847 T5148 oasc.RequestHandlers.initHandlersFromConfig
> created standard: solr.StandardRequestHandler
>  [junit4]   2> 1765851 T5174 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy:
> minMergeSize=0, mergeFactor=10, maxMergeSize=892100269,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>  [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>  [junit4]   2>
> commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede
> ),segFN=segments_1,generation=1}
>  [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>  [junit4]   2> 1765855 T5174 oass.SolrIndexSearcher.<init> Opening
> Searcher@290c4ce6 main
>  [junit4]   2> 1765852 T5190
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765857 T5190
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765858 T5190 oass.OpenExchangeRatesOrgProvider.reload
> Reloading exchange rates from open-exchange-rates.json
>  [junit4]   2> 1765858 T5148 oasc.RequestHandlers.initHandlersFromConfig
> created /get: solr.RealTimeGetHandler
>  [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig
> adding lazy requestHandler: solr.ReplicationHandler
>  [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig
> created /replication: solr.ReplicationHandler
>  [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig
> created /update: solr.UpdateRequestHandler
>  [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/: org.apache.solr.handler.admin.AdminHandlers
>  [junit4]   2> 1765861 T5148 oasc.RequestHandlers.initHandlersFromConfig
> created /admin/file: solr.admin.ShowFileRequestHandler
>  [junit4]   2> 1765865 T5148 oashl.XMLLoader.init
> xsltCacheLifetimeSeconds=60
>  [junit4]   2> 1765865 T5201 oasc.SolrCore.registerSearcher
> [solrj_collection_shard1_replica2] Registered new searcher Searcher@290c4ce6main{StandardDirectoryReader(segments_1:1:nrt)}
>  [junit4]   2> 1765869 T5174 oasc.CoreContainer.registerCore registering
> core: solrj_collection_shard1_replica2
>  [junit4]   2> 1765870 T5174 oasc.ZkController.register Register replica -
> core:solrj_collection_shard1_replica2 address:http://127.0.0.1:54120collection:solrj_collection shard:shard1
>  [junit4]   2> 1765872 T5190
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key
> IMPORTANT NOTE
>  [junit4]   2> 1765873 T5190
> oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected
> key, got STRING
>  [junit4]   2> 1765873 T5190 oasc.SolrCore.initDirectoryFactory
> org.apache.solr.core.MockDirectoryFactory
>  [junit4]   2> 1765874 T5190 oasc.SolrCore.<init>
> [solrj_collection_shard2_replica2] Opening new SolrCore at
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/,
> dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/
>  [junit4]   2> 1765875 T5190 oasc.JmxMonitoredMap.<init> JMX monitoring is
> enabled. Adding Solr mbeans to JMX Server:
> com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>  [junit4]   2> 1765876 T5174 oascc.SolrZkClient.makePath makePath:
> /collections/solrj_collection/leader_elect/shard1/election
>  [junit4]   2> 1765876 T5190 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data
>  [junit4]   2> 1765878 T5190 oasc.SolrCore.getNewIndexDir New index
> directory detected: old=null
> new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index/
>  [junit4]   2> 1765878 T5190 oasc.SolrCore.initIndex WARN
> [solrj_collection_shard2_replica2] Solr index directory
> './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index'
> doesn't exist. Creating new index...
>  [junit4]   2> 1765878 T5190 oasc.CachingDirectoryFactory.get return new
> directory for
> ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index
>  [junit4]   2> 1765879 T5190 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy:
> minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=false, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>  [junit4]   2> 1765879 T5148 oasu.CommitTracker.<init> Hard AutoCommit:
> disabled
>  [junit4]   2> 1765880 T5148 oasu.CommitTracker.<init> Soft AutoCommit:
> disabled
>  [junit4]   2> 1765881 T5148 oasu.RandomMergePolicy.<init>
> RandomMergePolicy wrapping class
> org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy:
> minMergeSize=0, mergeFactor=10, maxMergeSize=892100269,
> maxMergeSizeForForcedMerge=9223372036854775807,
> calibrateSizeByDeletes=true, maxMergeDocs=2147483647,
> maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>  [junit4]   2> 1765883 T5148 oasc.SolrDeletionPolicy.onInit
> SolrDeletionPolicy.onInit: commits: num=1
>  [junit4]   2>
> commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414
> ),segFN=segments_1,generation=1}
>  [junit4]   2> 1765885 T5148 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>  [junit4]   2> 1765886 T5148 oass.SolrIndexSearcher.<init> Opening
> Searcher@3dd43c40 main
>  [junit4]   2> 1765884 T5190 oasc.SolrDeletionPolicy.onCommit
> SolrDeletionPolicy.onCommit: commits: num=1
>  [junit4]   2>
> commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@40bfb0dflockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6da86f2c
> ),segFN=segments_1,generation=1}
>  [junit4]   2> 1765889 T5190 oasc.SolrDeletionPolicy.updateCommits newest
> commit generation = 1
>  [junit4]   2> 1765893 T5204 oasc.SolrCore.registerSearcher
> [solrj_collection_shard1_replica1] Registered new searcher Searcher@3dd43c40main{StandardDirectoryReader(segments_1:1:nrt)}
>  [junit4]   2> 1765896 T5148 oasc.CoreContainer.registerCore registering
> core: solrj_collection_shard1_replica1
>  [junit4]   2> 1765897 T5148 oasc.ZkController.register Register replica -
> core:solrj_collection_shard1_replica1 address:http://127.0.0.1:54114collection:solrj_collection shard:shard1
>  [junit4]   2> 1765896 T5190 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "nodistrib"
>  [junit4]   2> 1765899 T5190 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765896 T5126
> oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process
> for shard shard2
>  [junit4]   2> 1765901 T5190 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "dedupe"
>  [junit4]   2> 1765903 T5148 oascc.SolrZkClient.makePath makePath:
> /collections/solrj_collection/leader_elect/shard1/election
>  [junit4]   2> 1765909 T5190 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765911 T5190 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "stored_sig"
>  [junit4]   2> 1765912 T5190 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>  [junit4]   2> 1765913 T5190 oasup.UpdateRequestProcessorChain.init
> creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765913 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1765918 T5190 oasup.UpdateRequestProcessorChain.init
> inserting DistributedUpdateProcessorFactory into
> updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>  [junit4]   2> 1765919 T5190 oasc.SolrCore.loadUpdateProcessorChains no
> updateRequestProcessorChain defined as default, creating implicit default
>  [junit4]   2> 1765918 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1765920 T5134
> oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on
> path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>  [junit4]   2> 1765921 T5190 oasc.RequestHandlers.initHandlersFromConfig
> created standard: solr.StandardRequestHandler
>  [junit4]   2> 1765922 T5190 oasc.RequestHandlers.initHandlersFromConfig
> created /get: solr.RealTimeGetHandler
>  [junit4]   2> 1765923 T5190 oasc.RequestHandlers.initHandlersFromConfig
> adding lazy requestHandler: solr.ReplicationHandler
>  [junit4]   2> 1765924 T5190 oasc.RequestHandlers.initHandlersFromConfig
> created /replication: solr.ReplicationHandler
>  [junit4]   2> 1765925 T5126
> oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we
> see more replicas up for shard shard2: total=2 found=1 timeoutin=-8
>  [junit4]   2> 1765926 T5126
> oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for
> replicas to come up, but they are taking too long - assuming they won't
> come back till later
>  [junit4]   2> 1765927 T5126
> oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader -
> try and sync
>  [junit4]   2> ASYNC  NEW_CORE C3406 name=solrj_collection_shard2_replica1
> org.apache.solr.core.SolrCore@5da01731 url=
> http://127.0.0.1:54110/solrj_collection_shard2_replica1node=127.0.0.1:54110_
> C3406_STATE=coll:solrj_collection core:solrj_collection_shard2_replica1
> props:{state=down, base_url=http://127.0.0.1:54110,
> core=solrj_collection_shard2_replica1, node_name=127.0.0.1:54110_}
>  [junit4]   2> 1765929 T5126 C3406 P54110 oasc.SyncStrategy.sync Sync
> replicas to http://127.0.0.1:54110/solrj_collection_shard2_replica1/
>  [junit4]   2> 1765932 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync:
> core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 START
> replicas=[http://127.0.0.1:54123/solrj_collection_shard2_replica2/]
> nUpdates=100
>  [junit4]   2> 1765933 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync:
> core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 DONE.
>  We have no versions.  sync failed.
>  [junit4]   2> 1765934 T5126 C3406 P54110 oasc.SyncStrategy.syncReplicas
> Leader's attempt to sync with shard failed, moving to the next candidate
>  [junit4]   2> 1765935 T5126
> oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we
> have no versions - we can't sync in that case - we were active before, so
> become leader anyway
>  [junit4]   2> 1765936 T5126
> oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader:
> http://127.0.0.1:54110/solrj_collection_shard2_replica1/ shard2
>  [junit4]   2> 1765938 T5126 oascc.SolrZkClient.makePath makePath:
> /collections/solrj_collection/leaders/shard2
>  [junit4]   2> 1765926 T5190 oasc.RequestHandlers.initHandlersFromConfig
> created /update: solr.UpdateRequestHandler
>  [junit4]   2> 1765932 T5135 oascc.ZkStateReader.updateClusterState
> Updating cloud state from ZooKeeper...
>  [junit4]   2>
>
> [...truncated too long message...]
>
> ZkCmdExecutor.java:73)
>  [junit4]   2> at
> org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>  [junit4]   2> at
> org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>  [junit4]   2> at
> org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>  [junit4]   2>
>  [junit4]   2> 1991623 T5194 oascc.ZkStateReader$3.process WARN ZooKeeper
> watch triggered, but Solr cannot talk to ZK
>  [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN
>  org.apache.zookeeper.KeeperException$SessionExpiredException:
> KeeperErrorCode = Session expired for
> /collections/awholynewcollection_2/leader_elect/shard1/election
>  [junit4]   2> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>  [junit4]   2> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>  [junit4]   2> at
> org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>  [junit4]   2> at
> org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>  [junit4]   2> at
> org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>  [junit4]   2> at
> org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>  [junit4]   2> at
> org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>  [junit4]   2> at
> org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>  [junit4]   2> at
> org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>  [junit4]   2>
>  [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN
>  org.apache.zookeeper.KeeperException$SessionExpiredException:
> KeeperErrorCode = Session expired for
> /collections/awholynewcollection_0/leader_elect/shard3/election
>  [junit4]   2> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>  [junit4]   2> at
> org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>  [junit4]   2> at
> org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>  [junit4]   2> at
> org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>  [junit4]   2> at
> org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>  [junit4]   2> at
> org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>  [junit4]   2> at
> org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>  [junit4]   2> at
> org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>  [junit4]   2> at
> org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>  [junit4]   2> at
> org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>  [junit4]   2>
>  [junit4]   2> 1991804 T5180 oascc.ZkStateReader$3.process WARN ZooKeeper
> watch triggered, but Solr cannot talk to ZK
>  [junit4]   2> NOTE: test params are: codec=Lucene46:
> {range_facet_si=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136),
> range_facet_sl=PostingsFormat(name=Memory doPackFST= false),
> id=PostingsFormat(name=Memory doPackFST= false),
> a_si=Lucene41(blocksize=128), other_tl1=Lucene41(blocksize=128),
> text=PostingsFormat(name=MockRandom), multiDefault=Pulsing41(freqCutoff=1
> minBlockSize=29 maxBlockSize=136), intDefault=Lucene41(blocksize=128),
> timestamp=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136),
> a_t=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136),
> range_facet_l=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136),
> _version_=Lucene41(blocksize=128)}, docValues:{},
> sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=de_LU,
> timezone=CET
>  [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.7.0_51
> (64-bit)/cpus=2,threads=3,free=99625584,total=338677760
>  [junit4]   2> NOTE: All tests run in this JVM: [TestSolrJ,
> TestDistributedGrouping, TestFreeTextSuggestions,
> TestSchemaSimilarityResource, DirectUpdateHandlerTest,
> PreAnalyzedUpdateProcessorTest, TestCoreDiscovery,
> CoreMergeIndexesAdminHandlerTest, TestClassNameShortening, IndexSchemaTest,
> TestWordDelimiterFilterFactory, FullSolrCloudDistribCmdsTest,
> TestShardHandlerFactory, CircularListTest, TestRealTimeGet,
> SolrPluginUtilsTest, TestQueryUtils, TestExtendedDismaxParser,
> TestSchemaVersionResource, TestPostingsSolrHighlighter,
> UniqFieldsUpdateProcessorFactoryTest, MoreLikeThisHandlerTest,
> TestPhraseSuggestions, TestFuzzyAnalyzedSuggestions, UpdateParamsTest,
> TestInfoStreamLogging, BasicDistributedZk2Test, SyncSliceTest,
> TestPerFieldSimilarity, TestNRTOpen, BasicFunctionalityTest,
> TestAtomicUpdateErrorCases, SliceStateUpdateTest, JsonLoaderTest,
> TestMaxScoreQueryParser, BadIndexSchemaTest, PrimUtilsTest,
> TestMergePolicyConfig, BadCopyFieldTest, TestOmitPositions,
> TestRandomFaceting, RAMDirectoryFactoryTest, RequestHandlersTest,
> AliasIntegrationTest, HighlighterConfigTest, TestFileDictionaryLookup,
> TestLuceneMatchVersion, TestDefaultSearchFieldResource,
> TriLevelCompositeIdRoutingTest, UUIDFieldTest, TestSurroundQueryParser,
> StandardRequestHandlerTest, TestHashPartitioner, RegexBoostProcessorTest,
> TestBM25SimilarityFactory, SuggesterWFSTTest, DocumentBuilderTest,
> TestSolrDeletionPolicy2, SuggesterTest, OutputWriterTest, SimpleFacetsTest,
> FieldAnalysisRequestHandlerTest, TestFieldTypeResource,
> SolrIndexSplitterTest, TestPHPSerializedResponseWriter,
> SpellCheckComponentTest, UpdateRequestProcessorFactoryTest,
> TestComponentsName, TestDocBasedVersionConstraints, TestSolrCoreProperties,
> HdfsRecoveryZkTest, CoreAdminCreateDiscoverTest, TestZkChroot,
> TestSolrXmlPersistor, BJQParserTest, TestIndexSearcher,
> HdfsUnloadDistributedZkTest, TestPseudoReturnFields,
> TestCloudManagedSchemaAddField, TestPartialUpdateDeduplication,
> SuggesterFSTTest, CoreAdminHandlerTest, ChaosMonkeySafeLeaderTest,
> XmlUpdateRequestHandlerTest, TestAnalyzedSuggestions,
> LukeRequestHandlerTest, SuggesterTSTTest, CSVRequestHandlerTest,
> TestLFUCache, TestSolrQueryParserDefaultOperatorResource, StressHdfsTest,
> SpellPossibilityIteratorTest, ShardSplitTest, NotRequiredUniqueKeyTest,
> HdfsBasicDistributedZkTest, SearchHandlerTest, SimplePostToolTest,
> TestRandomDVFaceting, TestUtils, TestSystemIdResolver,
> CoreContainerCoreInitFailuresTest, TestSearchPerf, BlockDirectoryTest,
> DeleteShardTest, BasicDistributedZkTest, TestStressRecovery,
> TestNonNRTOpen, OverseerTest, SpatialFilterTest, ZkSolrClientTest,
> TestBinaryField, HighlighterTest, TestSolrDeletionPolicy1,
> TestQuerySenderListener, MigrateRouteKeyTest, URLClassifyProcessorTest,
> TestLRUCache, QueryResultKeyTest, ReturnFieldsTest, DateMathParserTest,
> JSONWriterTest, TestCSVResponseWriter, TestCustomSort,
> PathHierarchyTokenizerFactoryTest, ScriptEngineTest, RecoveryZkTest,
> ParsingFieldUpdateProcessorsTest, PolyFieldTest,
> TestSerializedLuceneMatchVersion, HdfsDirectoryTest, TestFastOutputStream,
> TimeZoneUtilsTest, TestMultiCoreConfBootstrap, QueryParsingTest,
> TestLMJelinekMercerSimilarityFactory, TestRandomMergePolicy,
> SoftAutoCommitTest, DirectSolrSpellCheckerTest, DeleteReplicaTest,
> AddBlockUpdateTest, TestIBSimilarityFactory,
> SchemaVersionSpecificBehaviorTest, SampleTest,
> TestHighFrequencyDictionaryFactory, TestSchemaNameResource,
> TestLMDirichletSimilarityFactory, ClusterStateUpdateTest,
> TestImplicitCoreProperties, TestFiltering, SortByFunctionTest,
> DebugComponentTest, TestCollationField, TestStressVersions,
> SolrTestCaseJ4Test, DocValuesMultiTest, HdfsChaosMonkeySafeLeaderTest,
> TestNumberUtils, TestDocSet, LegacyHTMLStripCharFilterTest,
> LeaderElectionTest, SolrRequestParserTest, QueryElevationComponentTest,
> DefaultValueUpdateProcessorTest, ResourceLoaderTest,
> TestQuerySenderNoQuery, TestCloudManagedSchema, ConnectionManagerTest,
> WordBreakSolrSpellCheckerTest, PingRequestHandlerTest, TestRangeQuery,
> DistanceFunctionTest, NoCacheHeaderTest, UnloadDistributedZkTest,
> CollectionsAPIDistributedZkTest]
>  [junit4] Completed in 238.84s, 1 test, 1 error <<< FAILURES!
>
> [...truncated 549 lines...]
> BUILD FAILED
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:459: The
> following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:439: The
> following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The
> following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/extra-targets.xml:37: The
> following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:189: The
> following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:491:
> The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1308:
> The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:941:
> There were test failures: 354 suites, 1540 tests, 1 error, 40 ignored (6
> assumptions)
>
> Total time: 85 minutes 3 seconds
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops
> -XX:+UseSerialGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>
>
>


-- 
- Mark

Re: [JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 1211 - Still Failing!

Posted by Mark Miller <ma...@gmail.com>.
Well, I thought it was https://issues.apache.org/jira/browse/SOLR-4805, but it’s not that, it’s just similar. 

Some clean up that still needs to be done here. It does affect 4.6.1 as well.

- Mark

On Jan 19, 2014, at 12:02 PM, Mark Miller <ma...@gmail.com> wrote:

> Well, this isn’t good. It’s an old bug brought back to life - its trying to register the core in zk again on reload - this was previously fixed. Hope it’s not in 4.6.1.
> 
> - Mark
> 
> On Jan 18, 2014, at 11:16 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:
> 
>> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/1211/
>> Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
>> 
>> 1 tests failed.
>> FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch
>> 
>> Error Message:
>> reloadcollection the collection time out:60s
>> 
>> Stack Trace:
>> org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: reloadcollection the collection time out:60s
>> 	at __randomizedtesting.SeedInfo.seed([A836D24B8F744B19:29D05C53F82B2B25]:0)
>> 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
>> 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
>> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:765)
>> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
>> 	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:606)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:744)
>> 
>> 
>> 
>> 
>> Build Log:
>> [...truncated 10447 lines...]
>>  [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
>>  [junit4]   2> 1753662 T5108 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
>>  [junit4]   2> 1753672 T5108 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
>>  [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1390103593043
>>  [junit4]   2> 1753674 T5108 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>>  [junit4]   2> 1753675 T5109 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
>>  [junit4]   2> 1753776 T5108 oasc.ZkTestServer.run start zk server on port:54105
>>  [junit4]   2> 1753778 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1753786 T5115 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@692bd0f2 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1753787 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1753788 T5108 oascc.SolrZkClient.makePath makePath: /solr
>>  [junit4]   2> 1753800 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1753806 T5117 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@46284bce name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1753806 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1753807 T5108 oascc.SolrZkClient.makePath makePath: /collections/collection1
>>  [junit4]   2> 1753816 T5108 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
>>  [junit4]   2> 1753824 T5108 oascc.SolrZkClient.makePath makePath: /collections/control_collection
>>  [junit4]   2> 1753831 T5108 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
>>  [junit4]   2> 1753836 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>>  [junit4]   2> 1753837 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
>>  [junit4]   2> 1753850 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
>>  [junit4]   2> 1753851 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
>>  [junit4]   2> 1753861 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1753862 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1753872 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>>  [junit4]   2> 1753873 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
>>  [junit4]   2> 1753886 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>>  [junit4]   2> 1753887 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
>>  [junit4]   2> 1753898 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>>  [junit4]   2> 1753899 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
>>  [junit4]   2> 1753909 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>>  [junit4]   2> 1753911 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
>>  [junit4]   2> 1753921 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
>>  [junit4]   2> 1753922 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
>>  [junit4]   2> 1753930 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1753931 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1753941 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
>>  [junit4]   2> 1753942 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
>>  [junit4]   2> 1753953 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
>>  [junit4]   2> 1753954 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
>>  [junit4]   2> Using old style solr.xml
>>  [junit4]   2> 1753972 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1753978 T5119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@218b21e5 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1753978 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1753979 T5108 oascc.SolrZkClient.makePath makePath: /solr
>>  [junit4]   2> 1753986 T5110 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x143a8a1805b0002, likely client has closed socket
>>  [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
>>  [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>>  [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
>>  [junit4]   2> 
>>  [junit4]   2> 1753990 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1753999 T5121 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3cb1f3a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1754000 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1754003 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
>>  [junit4]   2> 1754005 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
>>  [junit4]   2> 1754018 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
>>  [junit4]   2> 1754021 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
>>  [junit4]   2> 1754036 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1754037 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1754045 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
>>  [junit4]   2> 1754046 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
>>  [junit4]   2> 1754061 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
>>  [junit4]   2> 1754063 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
>>  [junit4]   2> 1754071 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
>>  [junit4]   2> 1754073 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
>>  [junit4]   2> 1754086 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
>>  [junit4]   2> 1754087 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
>>  [junit4]   2> 1754105 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1754106 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1754115 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
>>  [junit4]   2> 1754116 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
>>  [junit4]   2> 1754128 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
>>  [junit4]   2> 1754129 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
>>  [junit4]   2> 1754139 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
>>  [junit4]   2> 1754140 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
>>  [junit4]   2> 1754561 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1754575 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54110
>>  [junit4]   2> 1754576 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1754576 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521
>>  [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/'
>>  [junit4]   2> 1754614 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
>>  [junit4]   2> 1754697 T5108 oasc.CoreContainer.<init> New CoreContainer 725528504
>>  [junit4]   2> 1754697 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/]
>>  [junit4]   2> 1754699 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1754713 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1754714 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1754714 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1754715 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>>  [junit4]   2> 1754715 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1754716 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1754721 T5132 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20826f36 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1754722 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1754728 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1754733 T5134 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ee56b7a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1754733 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1754737 T5108 oascc.SolrZkClient.makePath makePath: /overseer/queue
>>  [junit4]   2> 1754748 T5108 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
>>  [junit4]   2> 1754758 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes
>>  [junit4]   2> 1754764 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54110_
>>  [junit4]   2> 1754766 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54110_
>>  [junit4]   2> 1754776 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect
>>  [junit4]   2> 1754783 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
>>  [junit4]   2> 1754796 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
>>  [junit4]   2> 1754802 T5108 oasc.Overseer.start Overseer (id=91101829074190341-127.0.0.1:54110_-n_0000000000) starting
>>  [junit4]   2> 1754813 T5108 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
>>  [junit4]   2> 1754826 T5136 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
>>  [junit4]   2> 1754827 T5108 oascc.SolrZkClient.makePath makePath: /clusterstate.json
>>  [junit4]   2> 1754829 T5136 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
>>  [junit4]   2> 1754835 T5108 oascc.SolrZkClient.makePath makePath: /aliases.json
>>  [junit4]   2> 1754842 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1754851 T5135 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
>>  [junit4]   2> 1754860 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1754861 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1754862 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1754867 T5139 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5bbaf1af name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1754868 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1754871 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1754879 T5108 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
>>  [junit4]   2> 1755261 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1755265 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54114
>>  [junit4]   2> 1755268 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250
>>  [junit4]   2> 1755269 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/'
>>  [junit4]   2> 1755308 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
>>  [junit4]   2> 1755412 T5108 oasc.CoreContainer.<init> New CoreContainer 96924362
>>  [junit4]   2> 1755412 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/]
>>  [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1755429 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1755429 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1755430 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1755430 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>>  [junit4]   2> 1755430 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1755433 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1755438 T5150 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@486daebc name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1755439 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1755446 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1755451 T5152 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d708b92 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1755452 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1755471 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1756488 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54114_
>>  [junit4]   2> 1756493 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54114_
>>  [junit4]   2> 1756508 T5134 oascc.ZkStateReader$3.process Updating live nodes... (2)
>>  [junit4]   2> 1756508 T5139 oascc.ZkStateReader$3.process Updating live nodes... (2)
>>  [junit4]   2> 1756510 T5152 oascc.ZkStateReader$3.process Updating live nodes... (2)
>>  [junit4]   2> 1756538 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1756539 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1756946 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1756951 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54117
>>  [junit4]   2> 1756954 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911
>>  [junit4]   2> 1756955 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/'
>>  [junit4]   2> 1756992 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/solr.xml
>>  [junit4]   2> 1757060 T5108 oasc.CoreContainer.<init> New CoreContainer 1734112660
>>  [junit4]   2> 1757061 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/]
>>  [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1757074 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1757074 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1757075 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1757075 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>>  [junit4]   2> 1757075 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1757077 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1757085 T5164 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@672b7c8f name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1757086 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1757094 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1757099 T5166 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@81214c4 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1757100 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1757110 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1758123 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54117_
>>  [junit4]   2> 1758128 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54117_
>>  [junit4]   2> 1758152 T5152 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1758155 T5139 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1758154 T5134 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1758155 T5166 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1758174 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1758175 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1758552 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1758557 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54120
>>  [junit4]   2> 1758559 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547
>>  [junit4]   2> 1758561 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/'
>>  [junit4]   2> 1758598 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
>>  [junit4]   2> 1758704 T5108 oasc.CoreContainer.<init> New CoreContainer 76332827
>>  [junit4]   2> 1758705 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/]
>>  [junit4]   2> 1758706 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1758708 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1758721 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1758721 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1758722 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1758722 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>>  [junit4]   2> 1758722 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1758724 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1758728 T5178 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@345687f2 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1758730 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1758737 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1758743 T5180 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fde9546 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1758744 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1758758 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1759770 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54120_
>>  [junit4]   2> 1759776 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54120_
>>  [junit4]   2> 1759791 T5166 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1759792 T5139 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1759800 T5180 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1759801 T5152 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1759802 T5134 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1759829 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1759830 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1760202 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1760208 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54123
>>  [junit4]   2> 1760210 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1760211 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201
>>  [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/'
>>  [junit4]   2> 1760251 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
>>  [junit4]   2> 1760338 T5108 oasc.CoreContainer.<init> New CoreContainer 723834019
>>  [junit4]   2> 1760339 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/]
>>  [junit4]   2> 1760340 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1760353 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1760354 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1760354 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1760355 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>>  [junit4]   2> 1760355 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1760357 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1760361 T5192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f042f75 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1760363 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1760370 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1760375 T5194 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3326075a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1760375 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1760387 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1761398 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54123_
>>  [junit4]   2> 1761407 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54123_
>>  [junit4]   2> 1761415 T5180 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1761416 T5134 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1761417 T5194 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1761417 T5152 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1761416 T5139 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1761417 T5166 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1762088 T5147 oasha.CollectionsHandler.handleCreateAction Creating Collection : router.field=myOwnField&version=2&name=solrj_collection&action=CREATE&wt=javabin&router.name=compositeId&numShards=2&collection.configName=conf1&replicationFactor=2
>>  [junit4]   2> 1762091 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762094 T5136 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
>>  [junit4]   2> 	  "operation":"createcollection",
>>  [junit4]   2> 	  "fromApi":"true",
>>  [junit4]   2> 	  "name":"solrj_collection",
>>  [junit4]   2> 	  "replicationFactor":"2",
>>  [junit4]   2> 	  "collection.configName":"conf1",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "router.field":"myOwnField",
>>  [junit4]   2> 	  "router.name":"compositeId"}
>>  [junit4]   2> 1762094 T5136 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
>>  [junit4]   2> 	  "operation":"createcollection",
>>  [junit4]   2> 	  "fromApi":"true",
>>  [junit4]   2> 	  "name":"solrj_collection",
>>  [junit4]   2> 	  "replicationFactor":"2",
>>  [junit4]   2> 	  "collection.configName":"conf1",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "router.field":"myOwnField",
>>  [junit4]   2> 	  "router.name":"compositeId"}
>>  [junit4]   2> 1762095 T5136 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
>>  [junit4]   2> 1762095 T5136 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
>>  [junit4]   2> 1762107 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762112 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>>  [junit4]   2> 1762113 T5135 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
>>  [junit4]   2> 1762114 T5135 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
>>  [junit4]   2> 1762121 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762123 T5152 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762123 T5139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762124 T5194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762123 T5180 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762124 T5166 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762124 T5134 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762208 T5136 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
>>  [junit4]   2> 1762209 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:54114_
>>  [junit4]   2> 1762209 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:54120_
>>  [junit4]   2> 1762210 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:54110_
>>  [junit4]   2> 1762210 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:54123_
>>  [junit4]   2> 1762222 T5148 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
>>  [junit4]   2> 1762223 T5126 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
>>  [junit4]   2> 1762224 T5174 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
>>  [junit4]   2> 1762224 T5190 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
>>  [junit4]   2> 1762227 T5148 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1762228 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762228 T5126 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1762229 T5174 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1762229 T5190 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1762233 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>>  [junit4]   2> 1762236 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:54114",
>>  [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:54114_",
>>  [junit4]   2> 	  "shard":"shard1",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1762243 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762246 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:54110",
>>  [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:54110_",
>>  [junit4]   2> 	  "shard":"shard2",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1762254 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762257 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:54123",
>>  [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:54123_",
>>  [junit4]   2> 	  "shard":"shard2",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1762265 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762267 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:54120",
>>  [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:54120_",
>>  [junit4]   2> 	  "shard":"shard1",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1762276 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1762383 T5152 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762383 T5166 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762383 T5134 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762383 T5139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762383 T5194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1762383 T5180 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1763229 T5148 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
>>  [junit4]   2> 1763230 T5148 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
>>  [junit4]   2> 1763231 T5148 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1
>>  [junit4]   2> 1763231 T5148 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1763230 T5126 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
>>  [junit4]   2> 1763233 T5148 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1763233 T5126 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
>>  [junit4]   2> 1763234 T5126 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1
>>  [junit4]   2> 1763234 T5126 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1763231 T5190 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
>>  [junit4]   2> 1763231 T5174 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
>>  [junit4]   2> 1763236 T5126 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1763237 T5126 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1763236 T5190 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
>>  [junit4]   2> 1763238 T5190 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2
>>  [junit4]   2> 1763239 T5190 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1763233 T5148 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1763240 T5126 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/'
>>  [junit4]   2> 1763237 T5174 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
>>  [junit4]   2> 1763243 T5174 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2
>>  [junit4]   2> 1763243 T5174 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1763242 T5190 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1763245 T5190 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1763247 T5174 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1763250 T5174 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1763249 T5148 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/'
>>  [junit4]   2> 1763269 T5190 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/'
>>  [junit4]   2> 1763277 T5174 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/'
>>  [junit4]   2> 1763420 T5148 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1763477 T5126 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1763482 T5190 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1763477 T5174 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1763684 T5174 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1763688 T5174 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1763689 T5126 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1763703 T5174 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
>>  [junit4]   2> 1763718 T5126 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1763735 T5126 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
>>  [junit4]   2> 1763767 T5148 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1763798 T5190 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1763809 T5148 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1763813 T5190 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1763838 T5190 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
>>  [junit4]   2> 1763843 T5148 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
>>  [junit4]   2> 1765449 T5126 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765457 T5126 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765458 T5126 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1765534 T5126 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1765545 T5126 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1765568 T5148 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765568 T5148 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765599 T5174 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765604 T5174 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765605 T5174 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1765612 T5174 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1765601 T5148 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1765635 T5174 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1765649 T5148 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1765665 T5148 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1765675 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765685 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765693 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765698 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765699 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765699 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765704 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765705 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765705 T5126 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1765706 T5126 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/
>>  [junit4]   2> 1765707 T5126 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>>  [junit4]   2> 1765706 T5190 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765712 T5174 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765713 T5190 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1765711 T5126 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data
>>  [junit4]   2> 1765721 T5174 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765718 T5190 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1765730 T5190 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1765735 T5174 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765722 T5126 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index/
>>  [junit4]   2> 1765739 T5126 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
>>  [junit4]   2> 1765739 T5126 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index
>>  [junit4]   2> 1765740 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>>  [junit4]   2> 1765740 T5190 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1765743 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765746 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765749 T5174 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765745 T5148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765756 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765757 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765758 T5174 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1765758 T5174 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/
>>  [junit4]   2> 1765758 T5174 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>>  [junit4]   2> 1765748 T5126 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6),segFN=segments_1,generation=1}
>>  [junit4]   2> 1765760 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1765761 T5174 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data
>>  [junit4]   2> 1765768 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>>  [junit4]   2> 1765768 T5148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765769 T5174 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index/
>>  [junit4]   2> 1765770 T5174 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
>>  [junit4]   2> 1765771 T5174 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index
>>  [junit4]   2> 1765773 T5174 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>>  [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede),segFN=segments_1,generation=1}
>>  [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1765778 T5148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765769 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765780 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>>  [junit4]   2> 1765781 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765783 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765784 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>>  [junit4]   2> 1765782 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765785 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765786 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765786 T5174 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>>  [junit4]   2> 1765788 T5174 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>>  [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>>  [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>>  [junit4]   2> 1765791 T5174 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>>  [junit4]   2> 1765785 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765792 T5148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765784 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>>  [junit4]   2> 1765792 T5174 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>>  [junit4]   2> 1765795 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765795 T5174 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>>  [junit4]   2> 1765797 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765799 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765799 T5148 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1765797 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765800 T5126 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>>  [junit4]   2> 1765800 T5148 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/
>>  [junit4]   2> 1765802 T5126 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>>  [junit4]   2> 1765802 T5190 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765798 T5174 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>>  [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>>  [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>>  [junit4]   2> 1765804 T5126 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>>  [junit4]   2> 1765802 T5148 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>>  [junit4]   2> 1765807 T5126 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>>  [junit4]   2> 1765809 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>>  [junit4]   2> 1765810 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>>  [junit4]   2> 1765813 T5148 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data
>>  [junit4]   2> 1765814 T5148 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index/
>>  [junit4]   2> 1765814 T5148 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
>>  [junit4]   2> 1765816 T5126 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>>  [junit4]   2> 1765820 T5148 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index
>>  [junit4]   2> 1765820 T5148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>>  [junit4]   2> 1765817 T5174 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>>  [junit4]   2> 1765823 T5190 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1765823 T5126 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>>  [junit4]   2> 1765825 T5126 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>>  [junit4]   2> 1765826 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>>  [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6),segFN=segments_1,generation=1}
>>  [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1765828 T5126 oass.SolrIndexSearcher.<init> Opening Searcher@190a2642 main
>>  [junit4]   2> 1765829 T5148 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414),segFN=segments_1,generation=1}
>>  [junit4]   2> 1765830 T5148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1765834 T5200 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica1] Registered new searcher Searcher@190a2642 main{StandardDirectoryReader(segments_1:1:nrt)}
>>  [junit4]   2> 1765837 T5126 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica1
>>  [junit4]   2> 1765837 T5126 oasc.ZkController.register Register replica - core:solrj_collection_shard2_replica1 address:http://127.0.0.1:54110 collection:solrj_collection shard:shard2
>>  [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>>  [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765842 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>>  [junit4]   2> 1765844 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765844 T5126 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard2/election
>>  [junit4]   2> 1765841 T5190 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765845 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765846 T5148 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>>  [junit4]   2> 1765842 T5174 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>>  [junit4]   2> 1765848 T5174 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>>  [junit4]   2> 1765847 T5148 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>>  [junit4]   2> 1765851 T5174 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>>  [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede),segFN=segments_1,generation=1}
>>  [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1765855 T5174 oass.SolrIndexSearcher.<init> Opening Searcher@290c4ce6 main
>>  [junit4]   2> 1765852 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765857 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765858 T5190 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1765858 T5148 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>>  [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>>  [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>>  [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>>  [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>>  [junit4]   2> 1765861 T5148 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>>  [junit4]   2> 1765865 T5148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>>  [junit4]   2> 1765865 T5201 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica2] Registered new searcher Searcher@290c4ce6 main{StandardDirectoryReader(segments_1:1:nrt)}
>>  [junit4]   2> 1765869 T5174 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica2
>>  [junit4]   2> 1765870 T5174 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica2 address:http://127.0.0.1:54120 collection:solrj_collection shard:shard1
>>  [junit4]   2> 1765872 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1765873 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1765873 T5190 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1765874 T5190 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/
>>  [junit4]   2> 1765875 T5190 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>>  [junit4]   2> 1765876 T5174 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
>>  [junit4]   2> 1765876 T5190 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data
>>  [junit4]   2> 1765878 T5190 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index/
>>  [junit4]   2> 1765878 T5190 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
>>  [junit4]   2> 1765878 T5190 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index
>>  [junit4]   2> 1765879 T5190 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>>  [junit4]   2> 1765879 T5148 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>>  [junit4]   2> 1765880 T5148 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>>  [junit4]   2> 1765881 T5148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>>  [junit4]   2> 1765883 T5148 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414),segFN=segments_1,generation=1}
>>  [junit4]   2> 1765885 T5148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1765886 T5148 oass.SolrIndexSearcher.<init> Opening Searcher@3dd43c40 main
>>  [junit4]   2> 1765884 T5190 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@40bfb0df lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6da86f2c),segFN=segments_1,generation=1}
>>  [junit4]   2> 1765889 T5190 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1765893 T5204 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@3dd43c40 main{StandardDirectoryReader(segments_1:1:nrt)}
>>  [junit4]   2> 1765896 T5148 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
>>  [junit4]   2> 1765897 T5148 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:54114 collection:solrj_collection shard:shard1
>>  [junit4]   2> 1765896 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>>  [junit4]   2> 1765899 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765896 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
>>  [junit4]   2> 1765901 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1765903 T5148 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
>>  [junit4]   2> 1765909 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765911 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1765912 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>>  [junit4]   2> 1765913 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765913 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1765918 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1765919 T5190 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>>  [junit4]   2> 1765918 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1765920 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1765921 T5190 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>>  [junit4]   2> 1765922 T5190 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>>  [junit4]   2> 1765923 T5190 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>>  [junit4]   2> 1765924 T5190 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>>  [junit4]   2> 1765925 T5126 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=-8
>>  [junit4]   2> 1765926 T5126 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
>>  [junit4]   2> 1765927 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>>  [junit4]   2> ASYNC  NEW_CORE C3406 name=solrj_collection_shard2_replica1 org.apache.solr.core.SolrCore@5da01731 url=http://127.0.0.1:54110/solrj_collection_shard2_replica1 node=127.0.0.1:54110_ C3406_STATE=coll:solrj_collection core:solrj_collection_shard2_replica1 props:{state=down, base_url=http://127.0.0.1:54110, core=solrj_collection_shard2_replica1, node_name=127.0.0.1:54110_}
>>  [junit4]   2> 1765929 T5126 C3406 P54110 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54110/solrj_collection_shard2_replica1/
>>  [junit4]   2> 1765932 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 START replicas=[http://127.0.0.1:54123/solrj_collection_shard2_replica2/] nUpdates=100
>>  [junit4]   2> 1765933 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 DONE.  We have no versions.  sync failed.
>>  [junit4]   2> 1765934 T5126 C3406 P54110 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
>>  [junit4]   2> 1765935 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
>>  [junit4]   2> 1765936 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54110/solrj_collection_shard2_replica1/ shard2
>>  [junit4]   2> 1765938 T5126 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leaders/shard2
>>  [junit4]   2> 1765926 T5190 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>>  [junit4]   2> 1765932 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>>  [junit4]   2> 
>> 
>> [...truncated too long message...]
>> 
>> ZkCmdExecutor.java:73)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>>  [junit4]   2> 
>>  [junit4]   2> 1991623 T5194 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>>  [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/awholynewcollection_2/leader_elect/shard1/election
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>>  [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>>  [junit4]   2> 
>>  [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/awholynewcollection_0/leader_elect/shard3/election
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>>  [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>>  [junit4]   2> 
>>  [junit4]   2> 1991804 T5180 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>>  [junit4]   2> NOTE: test params are: codec=Lucene46: {range_facet_si=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), range_facet_sl=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= false), a_si=Lucene41(blocksize=128), other_tl1=Lucene41(blocksize=128), text=PostingsFormat(name=MockRandom), multiDefault=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), intDefault=Lucene41(blocksize=128), timestamp=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), a_t=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), range_facet_l=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), _version_=Lucene41(blocksize=128)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=de_LU, timezone=CET
>>  [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.7.0_51 (64-bit)/cpus=2,threads=3,free=99625584,total=338677760
>>  [junit4]   2> NOTE: All tests run in this JVM: [TestSolrJ, TestDistributedGrouping, TestFreeTextSuggestions, TestSchemaSimilarityResource, DirectUpdateHandlerTest, PreAnalyzedUpdateProcessorTest, TestCoreDiscovery, CoreMergeIndexesAdminHandlerTest, TestClassNameShortening, IndexSchemaTest, TestWordDelimiterFilterFactory, FullSolrCloudDistribCmdsTest, TestShardHandlerFactory, CircularListTest, TestRealTimeGet, SolrPluginUtilsTest, TestQueryUtils, TestExtendedDismaxParser, TestSchemaVersionResource, TestPostingsSolrHighlighter, UniqFieldsUpdateProcessorFactoryTest, MoreLikeThisHandlerTest, TestPhraseSuggestions, TestFuzzyAnalyzedSuggestions, UpdateParamsTest, TestInfoStreamLogging, BasicDistributedZk2Test, SyncSliceTest, TestPerFieldSimilarity, TestNRTOpen, BasicFunctionalityTest, TestAtomicUpdateErrorCases, SliceStateUpdateTest, JsonLoaderTest, TestMaxScoreQueryParser, BadIndexSchemaTest, PrimUtilsTest, TestMergePolicyConfig, BadCopyFieldTest, TestOmitPositions, TestRandomFaceting, RAMDirectoryFactoryTest, RequestHandlersTest, AliasIntegrationTest, HighlighterConfigTest, TestFileDictionaryLookup, TestLuceneMatchVersion, TestDefaultSearchFieldResource, TriLevelCompositeIdRoutingTest, UUIDFieldTest, TestSurroundQueryParser, StandardRequestHandlerTest, TestHashPartitioner, RegexBoostProcessorTest, TestBM25SimilarityFactory, SuggesterWFSTTest, DocumentBuilderTest, TestSolrDeletionPolicy2, SuggesterTest, OutputWriterTest, SimpleFacetsTest, FieldAnalysisRequestHandlerTest, TestFieldTypeResource, SolrIndexSplitterTest, TestPHPSerializedResponseWriter, SpellCheckComponentTest, UpdateRequestProcessorFactoryTest, TestComponentsName, TestDocBasedVersionConstraints, TestSolrCoreProperties, HdfsRecoveryZkTest, CoreAdminCreateDiscoverTest, TestZkChroot, TestSolrXmlPersistor, BJQParserTest, TestIndexSearcher, HdfsUnloadDistributedZkTest, TestPseudoReturnFields, TestCloudManagedSchemaAddField, TestPartialUpdateDeduplication, SuggesterFSTTest, CoreAdminHandlerTest, ChaosMonkeySafeLeaderTest, XmlUpdateRequestHandlerTest, TestAnalyzedSuggestions, LukeRequestHandlerTest, SuggesterTSTTest, CSVRequestHandlerTest, TestLFUCache, TestSolrQueryParserDefaultOperatorResource, StressHdfsTest, SpellPossibilityIteratorTest, ShardSplitTest, NotRequiredUniqueKeyTest, HdfsBasicDistributedZkTest, SearchHandlerTest, SimplePostToolTest, TestRandomDVFaceting, TestUtils, TestSystemIdResolver, CoreContainerCoreInitFailuresTest, TestSearchPerf, BlockDirectoryTest, DeleteShardTest, BasicDistributedZkTest, TestStressRecovery, TestNonNRTOpen, OverseerTest, SpatialFilterTest, ZkSolrClientTest, TestBinaryField, HighlighterTest, TestSolrDeletionPolicy1, TestQuerySenderListener, MigrateRouteKeyTest, URLClassifyProcessorTest, TestLRUCache, QueryResultKeyTest, ReturnFieldsTest, DateMathParserTest, JSONWriterTest, TestCSVResponseWriter, TestCustomSort, PathHierarchyTokenizerFactoryTest, ScriptEngineTest, RecoveryZkTest, ParsingFieldUpdateProcessorsTest, PolyFieldTest, TestSerializedLuceneMatchVersion, HdfsDirectoryTest, TestFastOutputStream, TimeZoneUtilsTest, TestMultiCoreConfBootstrap, QueryParsingTest, TestLMJelinekMercerSimilarityFactory, TestRandomMergePolicy, SoftAutoCommitTest, DirectSolrSpellCheckerTest, DeleteReplicaTest, AddBlockUpdateTest, TestIBSimilarityFactory, SchemaVersionSpecificBehaviorTest, SampleTest, TestHighFrequencyDictionaryFactory, TestSchemaNameResource, TestLMDirichletSimilarityFactory, ClusterStateUpdateTest, TestImplicitCoreProperties, TestFiltering, SortByFunctionTest, DebugComponentTest, TestCollationField, TestStressVersions, SolrTestCaseJ4Test, DocValuesMultiTest, HdfsChaosMonkeySafeLeaderTest, TestNumberUtils, TestDocSet, LegacyHTMLStripCharFilterTest, LeaderElectionTest, SolrRequestParserTest, QueryElevationComponentTest, DefaultValueUpdateProcessorTest, ResourceLoaderTest, TestQuerySenderNoQuery, TestCloudManagedSchema, ConnectionManagerTest, WordBreakSolrSpellCheckerTest, PingRequestHandlerTest, TestRangeQuery, DistanceFunctionTest, NoCacheHeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
>>  [junit4] Completed in 238.84s, 1 test, 1 error <<< FAILURES!
>> 
>> [...truncated 549 lines...]
>> BUILD FAILED
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:459: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:439: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/extra-targets.xml:37: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:491: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1308: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:941: There were test failures: 354 suites, 1540 tests, 1 error, 40 ignored (6 assumptions)
>> 
>> Total time: 85 minutes 3 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
>> Archiving artifacts
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>> 
>> 
>> 
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
> 


Re: [JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 1211 - Still Failing!

Posted by Mark Miller <ma...@gmail.com>.
Well, this isn’t good. It’s an old bug brought back to life - its trying to register the core in zk again on reload - this was previously fixed. Hope it’s not in 4.6.1.

- Mark

On Jan 18, 2014, at 11:16 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:

> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/1211/
> Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
> 
> 1 tests failed.
> FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch
> 
> Error Message:
> reloadcollection the collection time out:60s
> 
> Stack Trace:
> org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: reloadcollection the collection time out:60s
> 	at __randomizedtesting.SeedInfo.seed([A836D24B8F744B19:29D05C53F82B2B25]:0)
> 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
> 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:765)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
> 	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:606)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:744)
> 
> 
> 
> 
> Build Log:
> [...truncated 10447 lines...]
>   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
>   [junit4]   2> 1753662 T5108 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
>   [junit4]   2> 1753672 T5108 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
>   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1390103593043
>   [junit4]   2> 1753674 T5108 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>   [junit4]   2> 1753675 T5109 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
>   [junit4]   2> 1753776 T5108 oasc.ZkTestServer.run start zk server on port:54105
>   [junit4]   2> 1753778 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1753786 T5115 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@692bd0f2 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1753787 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1753788 T5108 oascc.SolrZkClient.makePath makePath: /solr
>   [junit4]   2> 1753800 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1753806 T5117 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@46284bce name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1753806 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1753807 T5108 oascc.SolrZkClient.makePath makePath: /collections/collection1
>   [junit4]   2> 1753816 T5108 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
>   [junit4]   2> 1753824 T5108 oascc.SolrZkClient.makePath makePath: /collections/control_collection
>   [junit4]   2> 1753831 T5108 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
>   [junit4]   2> 1753836 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>   [junit4]   2> 1753837 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
>   [junit4]   2> 1753850 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
>   [junit4]   2> 1753851 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
>   [junit4]   2> 1753861 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1753862 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1753872 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>   [junit4]   2> 1753873 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
>   [junit4]   2> 1753886 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>   [junit4]   2> 1753887 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
>   [junit4]   2> 1753898 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>   [junit4]   2> 1753899 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
>   [junit4]   2> 1753909 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>   [junit4]   2> 1753911 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
>   [junit4]   2> 1753921 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
>   [junit4]   2> 1753922 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
>   [junit4]   2> 1753930 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1753931 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1753941 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
>   [junit4]   2> 1753942 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
>   [junit4]   2> 1753953 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
>   [junit4]   2> 1753954 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
>   [junit4]   2> Using old style solr.xml
>   [junit4]   2> 1753972 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1753978 T5119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@218b21e5 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1753978 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1753979 T5108 oascc.SolrZkClient.makePath makePath: /solr
>   [junit4]   2> 1753986 T5110 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x143a8a1805b0002, likely client has closed socket
>   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
>   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
>   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
>   [junit4]   2> 
>   [junit4]   2> 1753990 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1753999 T5121 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3cb1f3a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1754000 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1754003 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
>   [junit4]   2> 1754005 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
>   [junit4]   2> 1754018 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
>   [junit4]   2> 1754021 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
>   [junit4]   2> 1754036 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1754037 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1754045 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
>   [junit4]   2> 1754046 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
>   [junit4]   2> 1754061 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
>   [junit4]   2> 1754063 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
>   [junit4]   2> 1754071 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
>   [junit4]   2> 1754073 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
>   [junit4]   2> 1754086 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
>   [junit4]   2> 1754087 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
>   [junit4]   2> 1754105 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1754106 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1754115 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
>   [junit4]   2> 1754116 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
>   [junit4]   2> 1754128 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
>   [junit4]   2> 1754129 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
>   [junit4]   2> 1754139 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
>   [junit4]   2> 1754140 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
>   [junit4]   2> 1754561 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1754575 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54110
>   [junit4]   2> 1754576 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1754576 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521
>   [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/'
>   [junit4]   2> 1754614 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
>   [junit4]   2> 1754697 T5108 oasc.CoreContainer.<init> New CoreContainer 725528504
>   [junit4]   2> 1754697 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/]
>   [junit4]   2> 1754699 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1754713 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1754714 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1754714 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1754715 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>   [junit4]   2> 1754715 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1754716 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1754721 T5132 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20826f36 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1754722 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1754728 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1754733 T5134 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ee56b7a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1754733 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1754737 T5108 oascc.SolrZkClient.makePath makePath: /overseer/queue
>   [junit4]   2> 1754748 T5108 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
>   [junit4]   2> 1754758 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes
>   [junit4]   2> 1754764 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54110_
>   [junit4]   2> 1754766 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54110_
>   [junit4]   2> 1754776 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect
>   [junit4]   2> 1754783 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
>   [junit4]   2> 1754796 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
>   [junit4]   2> 1754802 T5108 oasc.Overseer.start Overseer (id=91101829074190341-127.0.0.1:54110_-n_0000000000) starting
>   [junit4]   2> 1754813 T5108 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
>   [junit4]   2> 1754826 T5136 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
>   [junit4]   2> 1754827 T5108 oascc.SolrZkClient.makePath makePath: /clusterstate.json
>   [junit4]   2> 1754829 T5136 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
>   [junit4]   2> 1754835 T5108 oascc.SolrZkClient.makePath makePath: /aliases.json
>   [junit4]   2> 1754842 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1754851 T5135 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
>   [junit4]   2> 1754860 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1754861 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1754862 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1754867 T5139 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5bbaf1af name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1754868 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1754871 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1754879 T5108 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
>   [junit4]   2> 1755261 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1755265 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54114
>   [junit4]   2> 1755268 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250
>   [junit4]   2> 1755269 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/'
>   [junit4]   2> 1755308 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
>   [junit4]   2> 1755412 T5108 oasc.CoreContainer.<init> New CoreContainer 96924362
>   [junit4]   2> 1755412 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/]
>   [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1755429 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1755429 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1755430 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1755430 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>   [junit4]   2> 1755430 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1755433 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1755438 T5150 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@486daebc name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1755439 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1755446 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1755451 T5152 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d708b92 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1755452 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1755471 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1756488 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54114_
>   [junit4]   2> 1756493 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54114_
>   [junit4]   2> 1756508 T5134 oascc.ZkStateReader$3.process Updating live nodes... (2)
>   [junit4]   2> 1756508 T5139 oascc.ZkStateReader$3.process Updating live nodes... (2)
>   [junit4]   2> 1756510 T5152 oascc.ZkStateReader$3.process Updating live nodes... (2)
>   [junit4]   2> 1756538 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1756539 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1756946 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1756951 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54117
>   [junit4]   2> 1756954 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911
>   [junit4]   2> 1756955 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/'
>   [junit4]   2> 1756992 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/solr.xml
>   [junit4]   2> 1757060 T5108 oasc.CoreContainer.<init> New CoreContainer 1734112660
>   [junit4]   2> 1757061 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/]
>   [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1757074 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1757074 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1757075 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1757075 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>   [junit4]   2> 1757075 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1757077 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1757085 T5164 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@672b7c8f name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1757086 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1757094 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1757099 T5166 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@81214c4 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1757100 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1757110 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1758123 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54117_
>   [junit4]   2> 1758128 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54117_
>   [junit4]   2> 1758152 T5152 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1758155 T5139 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1758154 T5134 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1758155 T5166 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1758174 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1758175 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1758552 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1758557 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54120
>   [junit4]   2> 1758559 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547
>   [junit4]   2> 1758561 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/'
>   [junit4]   2> 1758598 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
>   [junit4]   2> 1758704 T5108 oasc.CoreContainer.<init> New CoreContainer 76332827
>   [junit4]   2> 1758705 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/]
>   [junit4]   2> 1758706 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1758708 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1758721 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1758721 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1758722 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1758722 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>   [junit4]   2> 1758722 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1758724 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1758728 T5178 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@345687f2 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1758730 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1758737 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1758743 T5180 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fde9546 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1758744 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1758758 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1759770 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54120_
>   [junit4]   2> 1759776 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54120_
>   [junit4]   2> 1759791 T5166 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1759792 T5139 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1759800 T5180 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1759801 T5152 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1759802 T5134 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1759829 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1759830 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1760202 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1760208 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54123
>   [junit4]   2> 1760210 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1760211 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201
>   [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/'
>   [junit4]   2> 1760251 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
>   [junit4]   2> 1760338 T5108 oasc.CoreContainer.<init> New CoreContainer 723834019
>   [junit4]   2> 1760339 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/]
>   [junit4]   2> 1760340 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1760353 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1760354 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1760354 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1760355 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
>   [junit4]   2> 1760355 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1760357 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1760361 T5192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f042f75 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1760363 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1760370 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1760375 T5194 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3326075a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1760375 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1760387 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1761398 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54123_
>   [junit4]   2> 1761407 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54123_
>   [junit4]   2> 1761415 T5180 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1761416 T5134 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1761417 T5194 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1761417 T5152 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1761416 T5139 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1761417 T5166 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1762088 T5147 oasha.CollectionsHandler.handleCreateAction Creating Collection : router.field=myOwnField&version=2&name=solrj_collection&action=CREATE&wt=javabin&router.name=compositeId&numShards=2&collection.configName=conf1&replicationFactor=2
>   [junit4]   2> 1762091 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762094 T5136 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
>   [junit4]   2> 	  "operation":"createcollection",
>   [junit4]   2> 	  "fromApi":"true",
>   [junit4]   2> 	  "name":"solrj_collection",
>   [junit4]   2> 	  "replicationFactor":"2",
>   [junit4]   2> 	  "collection.configName":"conf1",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "router.field":"myOwnField",
>   [junit4]   2> 	  "router.name":"compositeId"}
>   [junit4]   2> 1762094 T5136 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
>   [junit4]   2> 	  "operation":"createcollection",
>   [junit4]   2> 	  "fromApi":"true",
>   [junit4]   2> 	  "name":"solrj_collection",
>   [junit4]   2> 	  "replicationFactor":"2",
>   [junit4]   2> 	  "collection.configName":"conf1",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "router.field":"myOwnField",
>   [junit4]   2> 	  "router.name":"compositeId"}
>   [junit4]   2> 1762095 T5136 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
>   [junit4]   2> 1762095 T5136 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
>   [junit4]   2> 1762107 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762112 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>   [junit4]   2> 1762113 T5135 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
>   [junit4]   2> 1762114 T5135 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
>   [junit4]   2> 1762121 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762123 T5152 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762123 T5139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762124 T5194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762123 T5180 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762124 T5166 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762124 T5134 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762208 T5136 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
>   [junit4]   2> 1762209 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:54114_
>   [junit4]   2> 1762209 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:54120_
>   [junit4]   2> 1762210 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:54110_
>   [junit4]   2> 1762210 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:54123_
>   [junit4]   2> 1762222 T5148 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
>   [junit4]   2> 1762223 T5126 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
>   [junit4]   2> 1762224 T5174 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
>   [junit4]   2> 1762224 T5190 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
>   [junit4]   2> 1762227 T5148 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1762228 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762228 T5126 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1762229 T5174 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1762229 T5190 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1762233 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>   [junit4]   2> 1762236 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:54114",
>   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:54114_",
>   [junit4]   2> 	  "shard":"shard1",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1762243 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762246 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:54110",
>   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:54110_",
>   [junit4]   2> 	  "shard":"shard2",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1762254 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762257 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:54123",
>   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:54123_",
>   [junit4]   2> 	  "shard":"shard2",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1762265 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762267 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:54120",
>   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:54120_",
>   [junit4]   2> 	  "shard":"shard1",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1762276 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1762383 T5152 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762383 T5166 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762383 T5134 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762383 T5139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762383 T5194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1762383 T5180 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1763229 T5148 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
>   [junit4]   2> 1763230 T5148 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
>   [junit4]   2> 1763231 T5148 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1
>   [junit4]   2> 1763231 T5148 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1763230 T5126 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
>   [junit4]   2> 1763233 T5148 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1763233 T5126 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
>   [junit4]   2> 1763234 T5126 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1
>   [junit4]   2> 1763234 T5126 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1763231 T5190 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
>   [junit4]   2> 1763231 T5174 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
>   [junit4]   2> 1763236 T5126 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1763237 T5126 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1763236 T5190 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
>   [junit4]   2> 1763238 T5190 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2
>   [junit4]   2> 1763239 T5190 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1763233 T5148 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1763240 T5126 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/'
>   [junit4]   2> 1763237 T5174 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
>   [junit4]   2> 1763243 T5174 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2
>   [junit4]   2> 1763243 T5174 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1763242 T5190 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1763245 T5190 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1763247 T5174 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1763250 T5174 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1763249 T5148 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/'
>   [junit4]   2> 1763269 T5190 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/'
>   [junit4]   2> 1763277 T5174 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/'
>   [junit4]   2> 1763420 T5148 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1763477 T5126 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1763482 T5190 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1763477 T5174 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1763684 T5174 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1763688 T5174 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1763689 T5126 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1763703 T5174 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
>   [junit4]   2> 1763718 T5126 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1763735 T5126 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
>   [junit4]   2> 1763767 T5148 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1763798 T5190 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1763809 T5148 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1763813 T5190 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1763838 T5190 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
>   [junit4]   2> 1763843 T5148 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
>   [junit4]   2> 1765449 T5126 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765457 T5126 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765458 T5126 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1765534 T5126 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1765545 T5126 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1765568 T5148 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765568 T5148 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765599 T5174 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765604 T5174 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765605 T5174 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1765612 T5174 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1765601 T5148 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1765635 T5174 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1765649 T5148 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1765665 T5148 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1765675 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765685 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765693 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765698 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765699 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765699 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765704 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765705 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765705 T5126 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1765706 T5126 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/
>   [junit4]   2> 1765707 T5126 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>   [junit4]   2> 1765706 T5190 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765712 T5174 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765713 T5190 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1765711 T5126 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data
>   [junit4]   2> 1765721 T5174 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765718 T5190 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1765730 T5190 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1765735 T5174 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765722 T5126 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index/
>   [junit4]   2> 1765739 T5126 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
>   [junit4]   2> 1765739 T5126 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index
>   [junit4]   2> 1765740 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>   [junit4]   2> 1765740 T5190 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1765743 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765746 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765749 T5174 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765745 T5148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765756 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765757 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765758 T5174 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1765758 T5174 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/
>   [junit4]   2> 1765758 T5174 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>   [junit4]   2> 1765748 T5126 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6),segFN=segments_1,generation=1}
>   [junit4]   2> 1765760 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1765761 T5174 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data
>   [junit4]   2> 1765768 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>   [junit4]   2> 1765768 T5148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765769 T5174 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index/
>   [junit4]   2> 1765770 T5174 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
>   [junit4]   2> 1765771 T5174 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index
>   [junit4]   2> 1765773 T5174 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>   [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede),segFN=segments_1,generation=1}
>   [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1765778 T5148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765769 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765780 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>   [junit4]   2> 1765781 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765783 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765784 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>   [junit4]   2> 1765782 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765785 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765786 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765786 T5174 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>   [junit4]   2> 1765788 T5174 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>   [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>   [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>   [junit4]   2> 1765791 T5174 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>   [junit4]   2> 1765785 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765792 T5148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765784 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>   [junit4]   2> 1765792 T5174 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>   [junit4]   2> 1765795 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765795 T5174 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>   [junit4]   2> 1765797 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765799 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765799 T5148 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1765797 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765800 T5126 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>   [junit4]   2> 1765800 T5148 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/
>   [junit4]   2> 1765802 T5126 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>   [junit4]   2> 1765802 T5190 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765798 T5174 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>   [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>   [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>   [junit4]   2> 1765804 T5126 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>   [junit4]   2> 1765802 T5148 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>   [junit4]   2> 1765807 T5126 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>   [junit4]   2> 1765809 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>   [junit4]   2> 1765810 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>   [junit4]   2> 1765813 T5148 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data
>   [junit4]   2> 1765814 T5148 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index/
>   [junit4]   2> 1765814 T5148 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
>   [junit4]   2> 1765816 T5126 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>   [junit4]   2> 1765820 T5148 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index
>   [junit4]   2> 1765820 T5148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>   [junit4]   2> 1765817 T5174 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>   [junit4]   2> 1765823 T5190 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1765823 T5126 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>   [junit4]   2> 1765825 T5126 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>   [junit4]   2> 1765826 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>   [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6),segFN=segments_1,generation=1}
>   [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1765828 T5126 oass.SolrIndexSearcher.<init> Opening Searcher@190a2642 main
>   [junit4]   2> 1765829 T5148 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414),segFN=segments_1,generation=1}
>   [junit4]   2> 1765830 T5148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1765834 T5200 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica1] Registered new searcher Searcher@190a2642 main{StandardDirectoryReader(segments_1:1:nrt)}
>   [junit4]   2> 1765837 T5126 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica1
>   [junit4]   2> 1765837 T5126 oasc.ZkController.register Register replica - core:solrj_collection_shard2_replica1 address:http://127.0.0.1:54110 collection:solrj_collection shard:shard2
>   [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>   [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765842 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>   [junit4]   2> 1765844 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765844 T5126 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard2/election
>   [junit4]   2> 1765841 T5190 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765845 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765846 T5148 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>   [junit4]   2> 1765842 T5174 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>   [junit4]   2> 1765848 T5174 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>   [junit4]   2> 1765847 T5148 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>   [junit4]   2> 1765851 T5174 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>   [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede),segFN=segments_1,generation=1}
>   [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1765855 T5174 oass.SolrIndexSearcher.<init> Opening Searcher@290c4ce6 main
>   [junit4]   2> 1765852 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765857 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765858 T5190 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1765858 T5148 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>   [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>   [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>   [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>   [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>   [junit4]   2> 1765861 T5148 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>   [junit4]   2> 1765865 T5148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>   [junit4]   2> 1765865 T5201 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica2] Registered new searcher Searcher@290c4ce6 main{StandardDirectoryReader(segments_1:1:nrt)}
>   [junit4]   2> 1765869 T5174 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica2
>   [junit4]   2> 1765870 T5174 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica2 address:http://127.0.0.1:54120 collection:solrj_collection shard:shard1
>   [junit4]   2> 1765872 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1765873 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1765873 T5190 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1765874 T5190 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/
>   [junit4]   2> 1765875 T5190 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
>   [junit4]   2> 1765876 T5174 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
>   [junit4]   2> 1765876 T5190 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data
>   [junit4]   2> 1765878 T5190 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index/
>   [junit4]   2> 1765878 T5190 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
>   [junit4]   2> 1765878 T5190 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index
>   [junit4]   2> 1765879 T5190 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
>   [junit4]   2> 1765879 T5148 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>   [junit4]   2> 1765880 T5148 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>   [junit4]   2> 1765881 T5148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
>   [junit4]   2> 1765883 T5148 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414),segFN=segments_1,generation=1}
>   [junit4]   2> 1765885 T5148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1765886 T5148 oass.SolrIndexSearcher.<init> Opening Searcher@3dd43c40 main
>   [junit4]   2> 1765884 T5190 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@40bfb0df lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6da86f2c),segFN=segments_1,generation=1}
>   [junit4]   2> 1765889 T5190 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1765893 T5204 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@3dd43c40 main{StandardDirectoryReader(segments_1:1:nrt)}
>   [junit4]   2> 1765896 T5148 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
>   [junit4]   2> 1765897 T5148 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:54114 collection:solrj_collection shard:shard1
>   [junit4]   2> 1765896 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>   [junit4]   2> 1765899 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765896 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
>   [junit4]   2> 1765901 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1765903 T5148 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
>   [junit4]   2> 1765909 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765911 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1765912 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>   [junit4]   2> 1765913 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765913 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1765918 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1765919 T5190 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>   [junit4]   2> 1765918 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1765920 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1765921 T5190 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>   [junit4]   2> 1765922 T5190 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>   [junit4]   2> 1765923 T5190 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>   [junit4]   2> 1765924 T5190 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>   [junit4]   2> 1765925 T5126 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=-8
>   [junit4]   2> 1765926 T5126 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
>   [junit4]   2> 1765927 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>   [junit4]   2> ASYNC  NEW_CORE C3406 name=solrj_collection_shard2_replica1 org.apache.solr.core.SolrCore@5da01731 url=http://127.0.0.1:54110/solrj_collection_shard2_replica1 node=127.0.0.1:54110_ C3406_STATE=coll:solrj_collection core:solrj_collection_shard2_replica1 props:{state=down, base_url=http://127.0.0.1:54110, core=solrj_collection_shard2_replica1, node_name=127.0.0.1:54110_}
>   [junit4]   2> 1765929 T5126 C3406 P54110 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54110/solrj_collection_shard2_replica1/
>   [junit4]   2> 1765932 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 START replicas=[http://127.0.0.1:54123/solrj_collection_shard2_replica2/] nUpdates=100
>   [junit4]   2> 1765933 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 DONE.  We have no versions.  sync failed.
>   [junit4]   2> 1765934 T5126 C3406 P54110 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
>   [junit4]   2> 1765935 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
>   [junit4]   2> 1765936 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54110/solrj_collection_shard2_replica1/ shard2
>   [junit4]   2> 1765938 T5126 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leaders/shard2
>   [junit4]   2> 1765926 T5190 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>   [junit4]   2> 1765932 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>   [junit4]   2> 
> 
> [...truncated too long message...]
> 
> ZkCmdExecutor.java:73)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>   [junit4]   2> 
>   [junit4]   2> 1991623 T5194 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>   [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/awholynewcollection_2/leader_elect/shard1/election
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>   [junit4]   2> 
>   [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/awholynewcollection_0/leader_elect/shard3/election
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>   [junit4]   2> 
>   [junit4]   2> 1991804 T5180 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>   [junit4]   2> NOTE: test params are: codec=Lucene46: {range_facet_si=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), range_facet_sl=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= false), a_si=Lucene41(blocksize=128), other_tl1=Lucene41(blocksize=128), text=PostingsFormat(name=MockRandom), multiDefault=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), intDefault=Lucene41(blocksize=128), timestamp=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), a_t=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), range_facet_l=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), _version_=Lucene41(blocksize=128)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=de_LU, timezone=CET
>   [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.7.0_51 (64-bit)/cpus=2,threads=3,free=99625584,total=338677760
>   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrJ, TestDistributedGrouping, TestFreeTextSuggestions, TestSchemaSimilarityResource, DirectUpdateHandlerTest, PreAnalyzedUpdateProcessorTest, TestCoreDiscovery, CoreMergeIndexesAdminHandlerTest, TestClassNameShortening, IndexSchemaTest, TestWordDelimiterFilterFactory, FullSolrCloudDistribCmdsTest, TestShardHandlerFactory, CircularListTest, TestRealTimeGet, SolrPluginUtilsTest, TestQueryUtils, TestExtendedDismaxParser, TestSchemaVersionResource, TestPostingsSolrHighlighter, UniqFieldsUpdateProcessorFactoryTest, MoreLikeThisHandlerTest, TestPhraseSuggestions, TestFuzzyAnalyzedSuggestions, UpdateParamsTest, TestInfoStreamLogging, BasicDistributedZk2Test, SyncSliceTest, TestPerFieldSimilarity, TestNRTOpen, BasicFunctionalityTest, TestAtomicUpdateErrorCases, SliceStateUpdateTest, JsonLoaderTest, TestMaxScoreQueryParser, BadIndexSchemaTest, PrimUtilsTest, TestMergePolicyConfig, BadCopyFieldTest, TestOmitPositions, TestRandomFaceting, RAMDirectoryFactoryTest, RequestHandlersTest, AliasIntegrationTest, HighlighterConfigTest, TestFileDictionaryLookup, TestLuceneMatchVersion, TestDefaultSearchFieldResource, TriLevelCompositeIdRoutingTest, UUIDFieldTest, TestSurroundQueryParser, StandardRequestHandlerTest, TestHashPartitioner, RegexBoostProcessorTest, TestBM25SimilarityFactory, SuggesterWFSTTest, DocumentBuilderTest, TestSolrDeletionPolicy2, SuggesterTest, OutputWriterTest, SimpleFacetsTest, FieldAnalysisRequestHandlerTest, TestFieldTypeResource, SolrIndexSplitterTest, TestPHPSerializedResponseWriter, SpellCheckComponentTest, UpdateRequestProcessorFactoryTest, TestComponentsName, TestDocBasedVersionConstraints, TestSolrCoreProperties, HdfsRecoveryZkTest, CoreAdminCreateDiscoverTest, TestZkChroot, TestSolrXmlPersistor, BJQParserTest, TestIndexSearcher, HdfsUnloadDistributedZkTest, TestPseudoReturnFields, TestCloudManagedSchemaAddField, TestPartialUpdateDeduplication, SuggesterFSTTest, CoreAdminHandlerTest, ChaosMonkeySafeLeaderTest, XmlUpdateRequestHandlerTest, TestAnalyzedSuggestions, LukeRequestHandlerTest, SuggesterTSTTest, CSVRequestHandlerTest, TestLFUCache, TestSolrQueryParserDefaultOperatorResource, StressHdfsTest, SpellPossibilityIteratorTest, ShardSplitTest, NotRequiredUniqueKeyTest, HdfsBasicDistributedZkTest, SearchHandlerTest, SimplePostToolTest, TestRandomDVFaceting, TestUtils, TestSystemIdResolver, CoreContainerCoreInitFailuresTest, TestSearchPerf, BlockDirectoryTest, DeleteShardTest, BasicDistributedZkTest, TestStressRecovery, TestNonNRTOpen, OverseerTest, SpatialFilterTest, ZkSolrClientTest, TestBinaryField, HighlighterTest, TestSolrDeletionPolicy1, TestQuerySenderListener, MigrateRouteKeyTest, URLClassifyProcessorTest, TestLRUCache, QueryResultKeyTest, ReturnFieldsTest, DateMathParserTest, JSONWriterTest, TestCSVResponseWriter, TestCustomSort, PathHierarchyTokenizerFactoryTest, ScriptEngineTest, RecoveryZkTest, ParsingFieldUpdateProcessorsTest, PolyFieldTest, TestSerializedLuceneMatchVersion, HdfsDirectoryTest, TestFastOutputStream, TimeZoneUtilsTest, TestMultiCoreConfBootstrap, QueryParsingTest, TestLMJelinekMercerSimilarityFactory, TestRandomMergePolicy, SoftAutoCommitTest, DirectSolrSpellCheckerTest, DeleteReplicaTest, AddBlockUpdateTest, TestIBSimilarityFactory, SchemaVersionSpecificBehaviorTest, SampleTest, TestHighFrequencyDictionaryFactory, TestSchemaNameResource, TestLMDirichletSimilarityFactory, ClusterStateUpdateTest, TestImplicitCoreProperties, TestFiltering, SortByFunctionTest, DebugComponentTest, TestCollationField, TestStressVersions, SolrTestCaseJ4Test, DocValuesMultiTest, HdfsChaosMonkeySafeLeaderTest, TestNumberUtils, TestDocSet, LegacyHTMLStripCharFilterTest, LeaderElectionTest, SolrRequestParserTest, QueryElevationComponentTest, DefaultValueUpdateProcessorTest, ResourceLoaderTest, TestQuerySenderNoQuery, TestCloudManagedSchema, ConnectionManagerTest, WordBreakSolrSpellCheckerTest, PingRequestHandlerTest, TestRangeQuery, DistanceFunctionTest, NoCacheHeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
>   [junit4] Completed in 238.84s, 1 test, 1 error <<< FAILURES!
> 
> [...truncated 549 lines...]
> BUILD FAILED
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:459: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:439: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/extra-targets.xml:37: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:491: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1308: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:941: There were test failures: 354 suites, 1540 tests, 1 error, 40 ignored (6 assumptions)
> 
> Total time: 85 minutes 3 seconds
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 1211 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/1211/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC

1 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
reloadcollection the collection time out:60s

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: reloadcollection the collection time out:60s
	at __randomizedtesting.SeedInfo.seed([A836D24B8F744B19:29D05C53F82B2B25]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:765)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
	at sun.reflect.GeneratedMethodAccessor39.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:744)




Build Log:
[...truncated 10447 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> 1753662 T5108 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1753672 T5108 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1390103593043
   [junit4]   2> 1753674 T5108 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1753675 T5109 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1753776 T5108 oasc.ZkTestServer.run start zk server on port:54105
   [junit4]   2> 1753778 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1753786 T5115 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@692bd0f2 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1753787 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1753788 T5108 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1753800 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1753806 T5117 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@46284bce name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1753806 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1753807 T5108 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1753816 T5108 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1753824 T5108 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1753831 T5108 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1753836 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1753837 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1753850 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1753851 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1753861 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1753862 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1753872 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1753873 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1753886 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1753887 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1753898 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1753899 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1753909 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1753911 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1753921 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1753922 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1753930 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1753931 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1753941 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1753942 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1753953 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1753954 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> Using old style solr.xml
   [junit4]   2> 1753972 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1753978 T5119 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@218b21e5 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1753978 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1753979 T5108 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1753986 T5110 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x143a8a1805b0002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:220)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:744)
   [junit4]   2> 
   [junit4]   2> 1753990 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1753999 T5121 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3cb1f3a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1754000 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1754003 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 1754005 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 1754018 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 1754021 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
   [junit4]   2> 1754036 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1754037 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1754045 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 1754046 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 1754061 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 1754063 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
   [junit4]   2> 1754071 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 1754073 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
   [junit4]   2> 1754086 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 1754087 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 1754105 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1754106 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1754115 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 1754116 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 1754128 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 1754129 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 1754139 T5108 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 1754140 T5108 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
   [junit4]   2> 1754561 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1754575 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54110
   [junit4]   2> 1754576 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1754576 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521
   [junit4]   2> 1754577 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/'
   [junit4]   2> 1754614 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
   [junit4]   2> 1754697 T5108 oasc.CoreContainer.<init> New CoreContainer 725528504
   [junit4]   2> 1754697 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/]
   [junit4]   2> 1754699 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1754700 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1754701 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1754702 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1754713 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1754714 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1754714 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1754715 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
   [junit4]   2> 1754715 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1754716 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1754721 T5132 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20826f36 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1754722 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1754728 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1754733 T5134 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ee56b7a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1754733 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1754737 T5108 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1754748 T5108 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1754758 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1754764 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54110_
   [junit4]   2> 1754766 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54110_
   [junit4]   2> 1754776 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1754783 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1754796 T5108 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1754802 T5108 oasc.Overseer.start Overseer (id=91101829074190341-127.0.0.1:54110_-n_0000000000) starting
   [junit4]   2> 1754813 T5108 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1754826 T5136 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1754827 T5108 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1754829 T5136 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 1754835 T5108 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1754842 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1754851 T5135 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1754860 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1754861 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1754862 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1754867 T5139 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5bbaf1af name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1754868 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1754871 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1754879 T5108 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1755261 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1755265 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54114
   [junit4]   2> 1755268 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1755268 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250
   [junit4]   2> 1755269 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/'
   [junit4]   2> 1755308 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
   [junit4]   2> 1755412 T5108 oasc.CoreContainer.<init> New CoreContainer 96924362
   [junit4]   2> 1755412 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/]
   [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1755415 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1755416 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1755417 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1755429 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1755429 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1755430 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1755430 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
   [junit4]   2> 1755430 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1755433 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1755438 T5150 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@486daebc name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1755439 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1755446 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1755451 T5152 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d708b92 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1755452 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1755471 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1756488 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54114_
   [junit4]   2> 1756493 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54114_
   [junit4]   2> 1756508 T5134 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1756508 T5139 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1756510 T5152 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1756538 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1756539 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1756946 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1756951 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54117
   [junit4]   2> 1756954 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1756954 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911
   [junit4]   2> 1756955 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/'
   [junit4]   2> 1756992 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/solr.xml
   [junit4]   2> 1757060 T5108 oasc.CoreContainer.<init> New CoreContainer 1734112660
   [junit4]   2> 1757061 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390103595911/]
   [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1757062 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1757063 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1757064 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1757074 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1757074 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1757075 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1757075 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
   [junit4]   2> 1757075 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1757077 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1757085 T5164 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@672b7c8f name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1757086 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1757094 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1757099 T5166 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@81214c4 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1757100 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1757110 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1758123 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54117_
   [junit4]   2> 1758128 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54117_
   [junit4]   2> 1758152 T5152 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1758155 T5139 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1758154 T5134 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1758155 T5166 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1758174 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1758175 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1758552 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1758557 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54120
   [junit4]   2> 1758559 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1758560 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547
   [junit4]   2> 1758561 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/'
   [junit4]   2> 1758598 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
   [junit4]   2> 1758704 T5108 oasc.CoreContainer.<init> New CoreContainer 76332827
   [junit4]   2> 1758705 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/]
   [junit4]   2> 1758706 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1758708 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1758709 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1758710 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1758721 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1758721 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1758722 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1758722 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
   [junit4]   2> 1758722 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1758724 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1758728 T5178 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@345687f2 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1758730 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1758737 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1758743 T5180 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@fde9546 name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1758744 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1758758 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1759770 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54120_
   [junit4]   2> 1759776 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54120_
   [junit4]   2> 1759791 T5166 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1759792 T5139 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1759800 T5180 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1759801 T5152 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1759802 T5134 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1759829 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1759830 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1760202 T5108 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1760208 T5108 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54123
   [junit4]   2> 1760210 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1760211 T5108 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201
   [junit4]   2> 1760212 T5108 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/'
   [junit4]   2> 1760251 T5108 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
   [junit4]   2> 1760338 T5108 oasc.CoreContainer.<init> New CoreContainer 723834019
   [junit4]   2> 1760339 T5108 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/]
   [junit4]   2> 1760340 T5108 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1760341 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1760342 T5108 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1760343 T5108 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1760353 T5108 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1760354 T5108 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1760354 T5108 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1760355 T5108 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:54105/solr
   [junit4]   2> 1760355 T5108 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1760357 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1760361 T5192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f042f75 name:ZooKeeperConnection Watcher:127.0.0.1:54105 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1760363 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1760370 T5108 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1760375 T5194 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3326075a name:ZooKeeperConnection Watcher:127.0.0.1:54105/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1760375 T5108 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1760387 T5108 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1761398 T5108 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54123_
   [junit4]   2> 1761407 T5108 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54123_
   [junit4]   2> 1761415 T5180 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1761416 T5134 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1761417 T5194 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1761417 T5152 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1761416 T5139 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1761417 T5166 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 1762080 T5108 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1762088 T5147 oasha.CollectionsHandler.handleCreateAction Creating Collection : router.field=myOwnField&version=2&name=solrj_collection&action=CREATE&wt=javabin&router.name=compositeId&numShards=2&collection.configName=conf1&replicationFactor=2
   [junit4]   2> 1762091 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762094 T5136 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 1762094 T5136 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 1762095 T5136 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
   [junit4]   2> 1762095 T5136 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
   [junit4]   2> 1762107 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762112 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1762113 T5135 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
   [junit4]   2> 1762114 T5135 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
   [junit4]   2> 1762121 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762123 T5152 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762123 T5139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762124 T5194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762123 T5180 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762124 T5166 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762124 T5134 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762208 T5136 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
   [junit4]   2> 1762209 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:54114_
   [junit4]   2> 1762209 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:54120_
   [junit4]   2> 1762210 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:54110_
   [junit4]   2> 1762210 T5136 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:54123_
   [junit4]   2> 1762222 T5148 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
   [junit4]   2> 1762223 T5126 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
   [junit4]   2> 1762224 T5174 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
   [junit4]   2> 1762224 T5190 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
   [junit4]   2> 1762227 T5148 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1762228 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762228 T5126 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1762229 T5174 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1762229 T5190 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1762233 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1762236 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54114",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54114_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1762243 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762246 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54110",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54110_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1762254 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762257 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54123",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54123_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1762265 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762267 T5135 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54120",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54120_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1762276 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1762383 T5152 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762383 T5166 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762383 T5134 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762383 T5139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762383 T5194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1762383 T5180 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 1763229 T5148 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
   [junit4]   2> 1763230 T5148 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solr.xml
   [junit4]   2> 1763231 T5148 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1
   [junit4]   2> 1763231 T5148 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1763230 T5126 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
   [junit4]   2> 1763233 T5148 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1763233 T5126 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solr.xml
   [junit4]   2> 1763234 T5126 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1
   [junit4]   2> 1763234 T5126 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1763231 T5190 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
   [junit4]   2> 1763231 T5174 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
   [junit4]   2> 1763236 T5126 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1763237 T5126 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1763236 T5190 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solr.xml
   [junit4]   2> 1763238 T5190 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2
   [junit4]   2> 1763239 T5190 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1763233 T5148 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1763240 T5126 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/'
   [junit4]   2> 1763237 T5174 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solr.xml
   [junit4]   2> 1763243 T5174 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2
   [junit4]   2> 1763243 T5174 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 1763242 T5190 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1763245 T5190 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1763247 T5174 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1763250 T5174 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 1763249 T5148 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/'
   [junit4]   2> 1763269 T5190 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/'
   [junit4]   2> 1763277 T5174 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/'
   [junit4]   2> 1763420 T5148 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1763477 T5126 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1763482 T5190 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1763477 T5174 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 1763684 T5174 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1763688 T5174 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1763689 T5126 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1763703 T5174 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
   [junit4]   2> 1763718 T5126 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1763735 T5126 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
   [junit4]   2> 1763767 T5148 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1763798 T5190 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1763809 T5148 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1763813 T5190 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1763838 T5190 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
   [junit4]   2> 1763843 T5148 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
   [junit4]   2> 1765449 T5126 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765457 T5126 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765458 T5126 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1765534 T5126 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1765545 T5126 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1765568 T5148 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765568 T5148 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765599 T5174 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765604 T5174 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765605 T5174 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1765612 T5174 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1765601 T5148 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1765635 T5174 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1765649 T5148 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1765665 T5148 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1765675 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765685 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765693 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765698 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765699 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765699 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765704 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765705 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765705 T5126 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1765706 T5126 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/
   [junit4]   2> 1765707 T5126 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
   [junit4]   2> 1765706 T5190 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765712 T5174 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765713 T5190 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1765711 T5126 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data
   [junit4]   2> 1765721 T5174 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765718 T5190 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1765730 T5190 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1765735 T5174 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765722 T5126 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index/
   [junit4]   2> 1765739 T5126 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1765739 T5126 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390103593521/solrj_collection_shard2_replica1/data/index
   [junit4]   2> 1765740 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
   [junit4]   2> 1765740 T5190 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1765743 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765746 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765749 T5174 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765745 T5148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765756 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765757 T5174 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765758 T5174 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1765758 T5174 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/
   [junit4]   2> 1765758 T5174 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
   [junit4]   2> 1765748 T5126 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6),segFN=segments_1,generation=1}
   [junit4]   2> 1765760 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1765761 T5174 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data
   [junit4]   2> 1765768 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1765768 T5148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765769 T5174 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index/
   [junit4]   2> 1765770 T5174 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1765771 T5174 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390103597547/solrj_collection_shard1_replica2/data/index
   [junit4]   2> 1765773 T5174 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
   [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede),segFN=segments_1,generation=1}
   [junit4]   2> 1765775 T5174 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1765778 T5148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765769 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765780 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1765781 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765780 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765783 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765782 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765784 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1765782 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765785 T5174 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765786 T5174 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765786 T5174 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1765788 T5174 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1765789 T5174 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1765791 T5174 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1765785 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765792 T5148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765784 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1765792 T5174 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1765795 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765795 T5174 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1765797 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765799 T5148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765799 T5148 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1765797 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765800 T5126 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1765800 T5148 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/
   [junit4]   2> 1765802 T5126 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1765802 T5190 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765798 T5174 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1765803 T5126 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1765804 T5126 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1765802 T5148 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
   [junit4]   2> 1765807 T5126 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1765809 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1765810 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1765813 T5148 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data
   [junit4]   2> 1765814 T5148 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index/
   [junit4]   2> 1765814 T5148 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1765816 T5126 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1765820 T5148 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390103594250/solrj_collection_shard1_replica1/data/index
   [junit4]   2> 1765820 T5148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
   [junit4]   2> 1765817 T5174 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1765823 T5190 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1765823 T5126 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1765825 T5126 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1765826 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7477e0e2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1e4480e6),segFN=segments_1,generation=1}
   [junit4]   2> 1765827 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1765828 T5126 oass.SolrIndexSearcher.<init> Opening Searcher@190a2642 main
   [junit4]   2> 1765829 T5148 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414),segFN=segments_1,generation=1}
   [junit4]   2> 1765830 T5148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1765834 T5200 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica1] Registered new searcher Searcher@190a2642 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1765837 T5126 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica1
   [junit4]   2> 1765837 T5126 oasc.ZkController.register Register replica - core:solrj_collection_shard2_replica1 address:http://127.0.0.1:54110 collection:solrj_collection shard:shard2
   [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1765841 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765842 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765843 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1765844 T5148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765844 T5126 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard2/election
   [junit4]   2> 1765841 T5190 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765845 T5148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765846 T5148 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1765842 T5174 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1765848 T5174 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1765847 T5148 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1765851 T5174 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a94a9c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4cd26ede),segFN=segments_1,generation=1}
   [junit4]   2> 1765854 T5174 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1765855 T5174 oass.SolrIndexSearcher.<init> Opening Searcher@290c4ce6 main
   [junit4]   2> 1765852 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765857 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765858 T5190 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1765858 T5148 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1765859 T5148 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1765860 T5148 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1765861 T5148 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1765865 T5148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1765865 T5201 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica2] Registered new searcher Searcher@290c4ce6 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1765869 T5174 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica2
   [junit4]   2> 1765870 T5174 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica2 address:http://127.0.0.1:54120 collection:solrj_collection shard:shard1
   [junit4]   2> 1765872 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1765873 T5190 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1765873 T5190 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1765874 T5190 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/
   [junit4]   2> 1765875 T5190 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@48580d0c
   [junit4]   2> 1765876 T5174 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
   [junit4]   2> 1765876 T5190 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data
   [junit4]   2> 1765878 T5190 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index/
   [junit4]   2> 1765878 T5190 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1765878 T5190 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390103599201/solrj_collection_shard2_replica2/data/index
   [junit4]   2> 1765879 T5190 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=11, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6052431603391322]
   [junit4]   2> 1765879 T5148 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1765880 T5148 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1765881 T5148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=892100269, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1765883 T5148 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@775d1479 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@75a7b414),segFN=segments_1,generation=1}
   [junit4]   2> 1765885 T5148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1765886 T5148 oass.SolrIndexSearcher.<init> Opening Searcher@3dd43c40 main
   [junit4]   2> 1765884 T5190 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@40bfb0df lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6da86f2c),segFN=segments_1,generation=1}
   [junit4]   2> 1765889 T5190 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1765893 T5204 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@3dd43c40 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1765896 T5148 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
   [junit4]   2> 1765897 T5148 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:54114 collection:solrj_collection shard:shard1
   [junit4]   2> 1765896 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1765899 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765896 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1765901 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1765903 T5148 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
   [junit4]   2> 1765909 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765911 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1765912 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1765913 T5190 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765913 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1765918 T5190 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1765919 T5190 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1765918 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1765920 T5134 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1765921 T5190 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1765922 T5190 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1765923 T5190 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1765924 T5190 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1765925 T5126 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=-8
   [junit4]   2> 1765926 T5126 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
   [junit4]   2> 1765927 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C3406 name=solrj_collection_shard2_replica1 org.apache.solr.core.SolrCore@5da01731 url=http://127.0.0.1:54110/solrj_collection_shard2_replica1 node=127.0.0.1:54110_ C3406_STATE=coll:solrj_collection core:solrj_collection_shard2_replica1 props:{state=down, base_url=http://127.0.0.1:54110, core=solrj_collection_shard2_replica1, node_name=127.0.0.1:54110_}
   [junit4]   2> 1765929 T5126 C3406 P54110 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54110/solrj_collection_shard2_replica1/
   [junit4]   2> 1765932 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 START replicas=[http://127.0.0.1:54123/solrj_collection_shard2_replica2/] nUpdates=100
   [junit4]   2> 1765933 T5126 C3406 P54110 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard2_replica1 url=http://127.0.0.1:54110 DONE.  We have no versions.  sync failed.
   [junit4]   2> 1765934 T5126 C3406 P54110 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
   [junit4]   2> 1765935 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1765936 T5126 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54110/solrj_collection_shard2_replica1/ shard2
   [junit4]   2> 1765938 T5126 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leaders/shard2
   [junit4]   2> 1765926 T5190 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1765932 T5135 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 

[...truncated too long message...]

ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 1991623 T5194 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/awholynewcollection_2/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 1991623 T5194 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/awholynewcollection_0/leader_elect/shard3/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
   [junit4]   2> 
   [junit4]   2> 1991804 T5180 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene46: {range_facet_si=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), range_facet_sl=PostingsFormat(name=Memory doPackFST= false), id=PostingsFormat(name=Memory doPackFST= false), a_si=Lucene41(blocksize=128), other_tl1=Lucene41(blocksize=128), text=PostingsFormat(name=MockRandom), multiDefault=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), intDefault=Lucene41(blocksize=128), timestamp=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), a_t=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), range_facet_l=Pulsing41(freqCutoff=1 minBlockSize=29 maxBlockSize=136), _version_=Lucene41(blocksize=128)}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=de_LU, timezone=CET
   [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.7.0_51 (64-bit)/cpus=2,threads=3,free=99625584,total=338677760
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrJ, TestDistributedGrouping, TestFreeTextSuggestions, TestSchemaSimilarityResource, DirectUpdateHandlerTest, PreAnalyzedUpdateProcessorTest, TestCoreDiscovery, CoreMergeIndexesAdminHandlerTest, TestClassNameShortening, IndexSchemaTest, TestWordDelimiterFilterFactory, FullSolrCloudDistribCmdsTest, TestShardHandlerFactory, CircularListTest, TestRealTimeGet, SolrPluginUtilsTest, TestQueryUtils, TestExtendedDismaxParser, TestSchemaVersionResource, TestPostingsSolrHighlighter, UniqFieldsUpdateProcessorFactoryTest, MoreLikeThisHandlerTest, TestPhraseSuggestions, TestFuzzyAnalyzedSuggestions, UpdateParamsTest, TestInfoStreamLogging, BasicDistributedZk2Test, SyncSliceTest, TestPerFieldSimilarity, TestNRTOpen, BasicFunctionalityTest, TestAtomicUpdateErrorCases, SliceStateUpdateTest, JsonLoaderTest, TestMaxScoreQueryParser, BadIndexSchemaTest, PrimUtilsTest, TestMergePolicyConfig, BadCopyFieldTest, TestOmitPositions, TestRandomFaceting, RAMDirectoryFactoryTest, RequestHandlersTest, AliasIntegrationTest, HighlighterConfigTest, TestFileDictionaryLookup, TestLuceneMatchVersion, TestDefaultSearchFieldResource, TriLevelCompositeIdRoutingTest, UUIDFieldTest, TestSurroundQueryParser, StandardRequestHandlerTest, TestHashPartitioner, RegexBoostProcessorTest, TestBM25SimilarityFactory, SuggesterWFSTTest, DocumentBuilderTest, TestSolrDeletionPolicy2, SuggesterTest, OutputWriterTest, SimpleFacetsTest, FieldAnalysisRequestHandlerTest, TestFieldTypeResource, SolrIndexSplitterTest, TestPHPSerializedResponseWriter, SpellCheckComponentTest, UpdateRequestProcessorFactoryTest, TestComponentsName, TestDocBasedVersionConstraints, TestSolrCoreProperties, HdfsRecoveryZkTest, CoreAdminCreateDiscoverTest, TestZkChroot, TestSolrXmlPersistor, BJQParserTest, TestIndexSearcher, HdfsUnloadDistributedZkTest, TestPseudoReturnFields, TestCloudManagedSchemaAddField, TestPartialUpdateDeduplication, SuggesterFSTTest, CoreAdminHandlerTest, ChaosMonkeySafeLeaderTest, XmlUpdateRequestHandlerTest, TestAnalyzedSuggestions, LukeRequestHandlerTest, SuggesterTSTTest, CSVRequestHandlerTest, TestLFUCache, TestSolrQueryParserDefaultOperatorResource, StressHdfsTest, SpellPossibilityIteratorTest, ShardSplitTest, NotRequiredUniqueKeyTest, HdfsBasicDistributedZkTest, SearchHandlerTest, SimplePostToolTest, TestRandomDVFaceting, TestUtils, TestSystemIdResolver, CoreContainerCoreInitFailuresTest, TestSearchPerf, BlockDirectoryTest, DeleteShardTest, BasicDistributedZkTest, TestStressRecovery, TestNonNRTOpen, OverseerTest, SpatialFilterTest, ZkSolrClientTest, TestBinaryField, HighlighterTest, TestSolrDeletionPolicy1, TestQuerySenderListener, MigrateRouteKeyTest, URLClassifyProcessorTest, TestLRUCache, QueryResultKeyTest, ReturnFieldsTest, DateMathParserTest, JSONWriterTest, TestCSVResponseWriter, TestCustomSort, PathHierarchyTokenizerFactoryTest, ScriptEngineTest, RecoveryZkTest, ParsingFieldUpdateProcessorsTest, PolyFieldTest, TestSerializedLuceneMatchVersion, HdfsDirectoryTest, TestFastOutputStream, TimeZoneUtilsTest, TestMultiCoreConfBootstrap, QueryParsingTest, TestLMJelinekMercerSimilarityFactory, TestRandomMergePolicy, SoftAutoCommitTest, DirectSolrSpellCheckerTest, DeleteReplicaTest, AddBlockUpdateTest, TestIBSimilarityFactory, SchemaVersionSpecificBehaviorTest, SampleTest, TestHighFrequencyDictionaryFactory, TestSchemaNameResource, TestLMDirichletSimilarityFactory, ClusterStateUpdateTest, TestImplicitCoreProperties, TestFiltering, SortByFunctionTest, DebugComponentTest, TestCollationField, TestStressVersions, SolrTestCaseJ4Test, DocValuesMultiTest, HdfsChaosMonkeySafeLeaderTest, TestNumberUtils, TestDocSet, LegacyHTMLStripCharFilterTest, LeaderElectionTest, SolrRequestParserTest, QueryElevationComponentTest, DefaultValueUpdateProcessorTest, ResourceLoaderTest, TestQuerySenderNoQuery, TestCloudManagedSchema, ConnectionManagerTest, WordBreakSolrSpellCheckerTest, PingRequestHandlerTest, TestRangeQuery, DistanceFunctionTest, NoCacheHeaderTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
   [junit4] Completed in 238.84s, 1 test, 1 error <<< FAILURES!

[...truncated 549 lines...]
BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:459: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:439: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/extra-targets.xml:37: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:491: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1308: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:941: There were test failures: 354 suites, 1540 tests, 1 error, 40 ignored (6 assumptions)

Total time: 85 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 1210 - Still Failing!

Posted by Mark Miller <ma...@gmail.com>.
Hmm…I must not have committed that change. I know I had done it locally recently, so figured it was causing this - it’s relatively new I think.

So, perhaps the 2 minute so timeout is long enough that a 330 second wait might not be enough…

I’ll try raising the wait in the test and see if it still trips.

- Mark

On Jan 18, 2014, at 6:35 PM, Mark Miller <ma...@gmail.com> wrote:

> Looks like it’s stuck in a socket read. I’ll turn the tests distributed so timeout back on.
> 
> - Mark
> 
> On Jan 18, 2014, at 4:22 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:
> 
>> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/1210/
>> Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
>> 
>> 1 tests failed.
>> FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch
>> 
>> Error Message:
>> There are still nodes recoverying - waited for 330 seconds
>> 
>> Stack Trace:
>> java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
>> 	at __randomizedtesting.SeedInfo.seed([E01973DE033FE27B:61FFFDC674608247]:0)
>> 	at org.junit.Assert.fail(Assert.java:93)
>> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:172)
>> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:131)
>> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:126)
>> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:772)
>> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
>> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
>> 	at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source)
>> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> 	at java.lang.reflect.Method.invoke(Method.java:606)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
>> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
>> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
>> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
>> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
>> 	at java.lang.Thread.run(Thread.java:744)
>> 
>> 
>> 
>> 
>> Build Log:
>> [...truncated 10429 lines...]
>>  [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
>>  [junit4]   2> 1719213 T5915 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /od_jjq/cq
>>  [junit4]   2> 1719222 T5915 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
>>  [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1390078610952
>>  [junit4]   2> 1719224 T5915 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>>  [junit4]   2> 1719225 T5916 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
>>  [junit4]   2> 1719327 T5915 oasc.ZkTestServer.run start zk server on port:53689
>>  [junit4]   2> 1719329 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1719338 T5922 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b2bf8c1 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1719338 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1719339 T5915 oascc.SolrZkClient.makePath makePath: /solr
>>  [junit4]   2> 1719353 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1719358 T5924 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31e8fc2a name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1719359 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1719359 T5915 oascc.SolrZkClient.makePath makePath: /collections/collection1
>>  [junit4]   2> 1719375 T5915 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
>>  [junit4]   2> 1719384 T5915 oascc.SolrZkClient.makePath makePath: /collections/control_collection
>>  [junit4]   2> 1719393 T5915 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
>>  [junit4]   2> 1719403 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>>  [junit4]   2> 1719404 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
>>  [junit4]   2> 1719421 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
>>  [junit4]   2> 1719422 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
>>  [junit4]   2> 1719432 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1719433 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1719444 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>>  [junit4]   2> 1719445 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
>>  [junit4]   2> 1719456 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>>  [junit4]   2> 1719457 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
>>  [junit4]   2> 1719469 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>>  [junit4]   2> 1719470 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
>>  [junit4]   2> 1719480 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>>  [junit4]   2> 1719481 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
>>  [junit4]   2> 1719492 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
>>  [junit4]   2> 1719493 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
>>  [junit4]   2> 1719503 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1719504 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1719513 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
>>  [junit4]   2> 1719514 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
>>  [junit4]   2> 1719524 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
>>  [junit4]   2> 1719526 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
>>  [junit4]   2> Using old style solr.xml
>>  [junit4]   2> 1719542 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1719547 T5926 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66930f41 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1719547 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1719548 T5915 oascc.SolrZkClient.makePath makePath: /solr
>>  [junit4]   2> 1719558 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1719562 T5928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3239c4d9 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1719562 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1719564 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
>>  [junit4]   2> 1719565 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
>>  [junit4]   2> 1719576 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
>>  [junit4]   2> 1719578 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
>>  [junit4]   2> 1719588 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1719589 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>>  [junit4]   2> 1719600 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
>>  [junit4]   2> 1719600 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
>>  [junit4]   2> 1719611 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
>>  [junit4]   2> 1719612 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
>>  [junit4]   2> 1719622 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
>>  [junit4]   2> 1719624 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
>>  [junit4]   2> 1719633 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
>>  [junit4]   2> 1719634 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
>>  [junit4]   2> 1719650 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1719651 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
>>  [junit4]   2> 1719683 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
>>  [junit4]   2> 1719684 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
>>  [junit4]   2> 1719694 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
>>  [junit4]   2> 1719695 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
>>  [junit4]   2> 1719706 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
>>  [junit4]   2> 1719707 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
>>  [junit4]   2> 1720156 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1720161 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53694
>>  [junit4]   2> 1720162 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1720163 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1720164 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458
>>  [junit4]   2> 1720164 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/'
>>  [junit4]   2> 1720208 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/solr.xml
>>  [junit4]   2> 1720327 T5915 oasc.CoreContainer.<init> New CoreContainer 370136241
>>  [junit4]   2> 1720328 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/]
>>  [junit4]   2> 1720330 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1720330 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1720331 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1720332 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1720333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1720333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1720334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1720334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1720335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1720350 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1720351 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1720352 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1720352 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>>  [junit4]   2> 1720353 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1720355 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1720360 T5939 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6954bbc7 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1720361 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1720370 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1720374 T5941 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f18b9ff name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1720375 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1720381 T5915 oascc.SolrZkClient.makePath makePath: /overseer/queue
>>  [junit4]   2> 1720394 T5915 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
>>  [junit4]   2> 1720406 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes
>>  [junit4]   2> 1720412 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53694_od_jjq%2Fcq
>>  [junit4]   2> 1720416 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53694_od_jjq%2Fcq
>>  [junit4]   2> 1720428 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect
>>  [junit4]   2> 1720436 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
>>  [junit4]   2> 1720451 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
>>  [junit4]   2> 1720460 T5915 oasc.Overseer.start Overseer (id=91100191848005637-127.0.0.1:53694_od_jjq%2Fcq-n_0000000000) starting
>>  [junit4]   2> 1720471 T5915 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
>>  [junit4]   2> 1720486 T5943 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
>>  [junit4]   2> 1720488 T5915 oascc.SolrZkClient.makePath makePath: /clusterstate.json
>>  [junit4]   2> 1720490 T5943 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
>>  [junit4]   2> 1720497 T5915 oascc.SolrZkClient.makePath makePath: /aliases.json
>>  [junit4]   2> 1720504 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1720515 T5942 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
>>  [junit4]   2> 1720523 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1720524 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1720527 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1720532 T5946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4fdc9238 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1720532 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1720536 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1720546 T5915 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
>>  [junit4]   2> 1721004 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1721012 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53698
>>  [junit4]   2> 1721013 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1721013 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1721014 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275
>>  [junit4]   2> 1721014 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/'
>>  [junit4]   2> 1721059 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solr.xml
>>  [junit4]   2> 1721165 T5915 oasc.CoreContainer.<init> New CoreContainer 2108103389
>>  [junit4]   2> 1721165 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/]
>>  [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1721186 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1721187 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1721187 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1721188 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>>  [junit4]   2> 1721189 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1721192 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1721207 T5957 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65d4e39d name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1721207 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1721220 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1721228 T5959 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4db4e5c6 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1721229 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1721243 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1722255 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53698_od_jjq%2Fcq
>>  [junit4]   2> 1722258 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53698_od_jjq%2Fcq
>>  [junit4]   2> 1722266 T5959 oascc.ZkStateReader$3.process Updating live nodes... (2)
>>  [junit4]   2> 1722266 T5941 oascc.ZkStateReader$3.process Updating live nodes... (2)
>>  [junit4]   2> 1722267 T5946 oascc.ZkStateReader$3.process Updating live nodes... (2)
>>  [junit4]   2> 1722281 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1722281 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1722625 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1722630 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53701
>>  [junit4]   2> 1722631 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011
>>  [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/'
>>  [junit4]   2> 1722668 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solr.xml
>>  [junit4]   2> 1722738 T5915 oasc.CoreContainer.<init> New CoreContainer 1501927636
>>  [junit4]   2> 1722739 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/]
>>  [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1722743 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1722743 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1722753 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1722753 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1722753 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1722754 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>>  [junit4]   2> 1722754 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1722755 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1722760 T5971 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51f4b436 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1722760 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1722767 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1722771 T5973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b11c2d8 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1722771 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1722782 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1723792 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53701_od_jjq%2Fcq
>>  [junit4]   2> 1723795 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53701_od_jjq%2Fcq
>>  [junit4]   2> 1723805 T5941 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1723806 T5946 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1723807 T5959 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1723807 T5973 oascc.ZkStateReader$3.process Updating live nodes... (3)
>>  [junit4]   2> 1723827 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1723828 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1724210 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1724215 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53704
>>  [junit4]   2> 1724216 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1724216 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1724217 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557
>>  [junit4]   2> 1724217 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/'
>>  [junit4]   2> 1724246 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solr.xml
>>  [junit4]   2> 1724331 T5915 oasc.CoreContainer.<init> New CoreContainer 1311775510
>>  [junit4]   2> 1724332 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/]
>>  [junit4]   2> 1724333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1724333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1724346 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1724347 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1724347 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1724347 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>>  [junit4]   2> 1724348 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1724349 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1724354 T5985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38426974 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1724354 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1724361 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1724364 T5987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22116710 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1724365 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1724377 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1725388 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53704_od_jjq%2Fcq
>>  [junit4]   2> 1725391 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53704_od_jjq%2Fcq
>>  [junit4]   2> 1725402 T5946 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1725402 T5941 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1725403 T5959 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1725403 T5987 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1725403 T5973 oascc.ZkStateReader$3.process Updating live nodes... (4)
>>  [junit4]   2> 1725420 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1725421 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1725880 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>>  [junit4]   2> 1725885 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53707
>>  [junit4]   2> 1725886 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>>  [junit4]   2> 1725886 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>>  [junit4]   2> 1725887 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150
>>  [junit4]   2> 1725888 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/'
>>  [junit4]   2> 1725922 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solr.xml
>>  [junit4]   2> 1725993 T5915 oasc.CoreContainer.<init> New CoreContainer 1201695941
>>  [junit4]   2> 1725994 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/]
>>  [junit4]   2> 1725995 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>>  [junit4]   2> 1725995 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>>  [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>>  [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>>  [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>>  [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>>  [junit4]   2> 1725997 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>>  [junit4]   2> 1725997 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>>  [junit4]   2> 1725998 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>>  [junit4]   2> 1726008 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>>  [junit4]   2> 1726008 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>>  [junit4]   2> 1726008 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>>  [junit4]   2> 1726009 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>>  [junit4]   2> 1726009 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>>  [junit4]   2> 1726010 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1726014 T5999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75451e3f name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1726015 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1726022 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>>  [junit4]   2> 1726028 T6001 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@509b80a4 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>>  [junit4]   2> 1726028 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>>  [junit4]   2> 1726040 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>>  [junit4]   2> 1727054 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53707_od_jjq%2Fcq
>>  [junit4]   2> 1727056 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53707_od_jjq%2Fcq
>>  [junit4]   2> 1727068 T5959 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1727068 T5946 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1727068 T5987 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1727068 T6001 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1727068 T5941 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1727068 T5973 oascc.ZkStateReader$3.process Updating live nodes... (5)
>>  [junit4]   2> 1727088 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>>  [junit4]   2> 1727088 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>>  [junit4]   2> 1727135 T5953 oasha.CollectionsHandler.handleCreateAction Creating Collection : version=2&numShards=2&action=CREATE&collection.configName=conf1&router.field=myOwnField&name=solrj_collection&replicationFactor=2&wt=javabin&router.name=compositeId
>>  [junit4]   2> 1727138 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727142 T5943 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
>>  [junit4]   2> 	  "operation":"createcollection",
>>  [junit4]   2> 	  "fromApi":"true",
>>  [junit4]   2> 	  "name":"solrj_collection",
>>  [junit4]   2> 	  "replicationFactor":"2",
>>  [junit4]   2> 	  "collection.configName":"conf1",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "router.field":"myOwnField",
>>  [junit4]   2> 	  "router.name":"compositeId"}
>>  [junit4]   2> 1727142 T5943 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
>>  [junit4]   2> 	  "operation":"createcollection",
>>  [junit4]   2> 	  "fromApi":"true",
>>  [junit4]   2> 	  "name":"solrj_collection",
>>  [junit4]   2> 	  "replicationFactor":"2",
>>  [junit4]   2> 	  "collection.configName":"conf1",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "router.field":"myOwnField",
>>  [junit4]   2> 	  "router.name":"compositeId"}
>>  [junit4]   2> 1727143 T5943 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
>>  [junit4]   2> 1727143 T5943 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
>>  [junit4]   2> 1727156 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727160 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>>  [junit4]   2> 1727161 T5942 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
>>  [junit4]   2> 1727162 T5942 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
>>  [junit4]   2> 1727169 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727172 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727172 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727172 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727172 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727172 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727172 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727257 T5943 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
>>  [junit4]   2> 1727258 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:53698_od_jjq%2Fcq
>>  [junit4]   2> 1727259 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:53707_od_jjq%2Fcq
>>  [junit4]   2> 1727260 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:53704_od_jjq%2Fcq
>>  [junit4]   2> 1727262 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:53701_od_jjq%2Fcq
>>  [junit4]   2> 1727267 T5954 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
>>  [junit4]   2> 1727271 T5979 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
>>  [junit4]   2> 1727273 T5995 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
>>  [junit4]   2> 1727273 T5969 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
>>  [junit4]   2> 1727276 T5954 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1727277 T5979 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1727277 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727279 T5969 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1727281 T5995 oasc.ZkController.waitForCoreNodeName look for our core node name
>>  [junit4]   2> 1727284 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>>  [junit4]   2> 1727287 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:53698/od_jjq/cq",
>>  [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:53698_od_jjq%2Fcq",
>>  [junit4]   2> 	  "shard":"shard1",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1727295 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727300 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:53704/od_jjq/cq",
>>  [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:53704_od_jjq%2Fcq",
>>  [junit4]   2> 	  "shard":"shard2",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1727310 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727314 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:53701/od_jjq/cq",
>>  [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:53701_od_jjq%2Fcq",
>>  [junit4]   2> 	  "shard":"shard2",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1727321 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727324 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>>  [junit4]   2> 	  "operation":"state",
>>  [junit4]   2> 	  "state":"down",
>>  [junit4]   2> 	  "base_url":"http://127.0.0.1:53707/od_jjq/cq",
>>  [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
>>  [junit4]   2> 	  "roles":null,
>>  [junit4]   2> 	  "node_name":"127.0.0.1:53707_od_jjq%2Fcq",
>>  [junit4]   2> 	  "shard":"shard1",
>>  [junit4]   2> 	  "shard_range":null,
>>  [junit4]   2> 	  "shard_state":"active",
>>  [junit4]   2> 	  "shard_parent":null,
>>  [junit4]   2> 	  "collection":"solrj_collection",
>>  [junit4]   2> 	  "numShards":"2",
>>  [junit4]   2> 	  "core_node_name":null}
>>  [junit4]   2> 1727332 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1727445 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727445 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727446 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727445 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727446 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1727446 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1728278 T5954 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
>>  [junit4]   2> 1728278 T5979 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
>>  [junit4]   2> 1728279 T5954 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solr.xml
>>  [junit4]   2> 1728279 T5979 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solr.xml
>>  [junit4]   2> 1728280 T5979 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solrj_collection_shard2_replica1
>>  [junit4]   2> 1728280 T5979 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1728281 T5969 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
>>  [junit4]   2> 1728280 T5954 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1
>>  [junit4]   2> 1728281 T5954 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1728282 T5969 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solr.xml
>>  [junit4]   2> 1728282 T5969 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2
>>  [junit4]   2> 1728282 T5969 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1728282 T5995 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
>>  [junit4]   2> 1728284 T5995 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solr.xml
>>  [junit4]   2> 1728284 T5979 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1728284 T5995 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2
>>  [junit4]   2> 1728284 T5954 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1728285 T5954 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1728285 T5995 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>>  [junit4]   2> 1728285 T5969 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1728286 T5969 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1728284 T5979 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1728287 T5995 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>>  [junit4]   2> 1728287 T5995 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>>  [junit4]   2> 1728289 T5954 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/'
>>  [junit4]   2> 1728291 T5995 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/'
>>  [junit4]   2> 1728292 T5969 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/'
>>  [junit4]   2> 1728295 T5979 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solrj_collection_shard2_replica1/'
>>  [junit4]   2> 1728399 T5954 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1728413 T5995 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1728426 T5969 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1728492 T5979 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>>  [junit4]   2> 1728547 T5969 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1728550 T5995 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1728554 T5969 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1728574 T5995 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1728577 T5969 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
>>  [junit4]   2> 1728610 T5995 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
>>  [junit4]   2> 1728640 T5954 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1728652 T5979 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>>  [junit4]   2> 1728653 T5954 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1728663 T5954 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
>>  [junit4]   2> 1728663 T5979 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>>  [junit4]   2> 1728715 T5979 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
>>  [junit4]   2> 1730003 T5954 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730003 T5954 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730004 T5954 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1730012 T5954 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1730018 T5954 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1730071 T5969 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730071 T5969 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730073 T5969 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1730090 T5969 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1730097 T5969 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1730128 T5995 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730128 T5995 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730129 T5995 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1730129 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730137 T5969 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730141 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730153 T5995 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1730158 T5995 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1730166 T5969 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730176 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730180 T5969 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730181 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730182 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730182 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730182 T5979 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730184 T5979 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>>  [junit4]   2> 1730184 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730184 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730185 T5969 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730186 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730186 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730187 T5954 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1730187 T5954 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/, dataDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/
>>  [junit4]   2> 1730188 T5954 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
>>  [junit4]   2> 1730185 T5979 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>>  [junit4]   2> 1730189 T5954 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data
>>  [junit4]   2> 1730189 T5954 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index/
>>  [junit4]   2> 1730190 T5954 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
>>  [junit4]   2> 1730190 T5954 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index
>>  [junit4]   2> 1730191 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
>>  [junit4]   2> 1730192 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730192 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730192 T5969 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1730193 T5969 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/, dataDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/
>>  [junit4]   2> 1730193 T5969 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
>>  [junit4]   2> 1730194 T5969 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data
>>  [junit4]   2> 1730195 T5969 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index/
>>  [junit4]   2> 1730195 T5969 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
>>  [junit4]   2> 1730197 T5969 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index
>>  [junit4]   2> 1730197 T5969 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
>>  [junit4]   2> 1730199 T5954 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@247327d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@392d8b48),segFN=segments_1,generation=1}
>>  [junit4]   2> 1730199 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1730199 T5969 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@434ea2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4d5719cb),segFN=segments_1,generation=1}
>>  [junit4]   2> 1730200 T5969 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1730200 T5979 oass.IndexSchema.readSchema default search field in schema is text
>>  [junit4]   2> 1730204 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>>  [junit4]   2> 1730204 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1730205 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1730206 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1730207 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1730207 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>>  [junit4]   2> 1730208 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1730209 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1730209 T5969 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>>  [junit4]   2> 1730204 T5979 oass.IndexSchema.readSchema unique key field: id
>>  [junit4]   2> 1730210 T5969 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>>  [junit4]   2> 1730212 T5969 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>>  [junit4]   2> 1730213 T5969 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>>  [junit4]   2> 1730213 T5969 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>>  [junit4]   2> 1730214 T5969 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>>  [junit4]   2> 1730215 T5969 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>>  [junit4]   2> 1730209 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>>  [junit4]   2> 1730216 T5969 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>>  [junit4]   2> 1730216 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1730217 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1730217 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1730218 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1730219 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>>  [junit4]   2> 1730220 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1730220 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1730221 T5954 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>>  [junit4]   2> 1730222 T5954 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>>  [junit4]   2> 1730222 T5954 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>>  [junit4]   2> 1730223 T5954 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>>  [junit4]   2> 1730223 T5954 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>>  [junit4]   2> 1730224 T5954 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>>  [junit4]   2> 1730227 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>>  [junit4]   2> 1730226 T5969 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>>  [junit4]   2> 1730228 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>>  [junit4]   2> 1730235 T5954 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>>  [junit4]   2> 1730240 T5969 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>>  [junit4]   2> 1730241 T5969 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>>  [junit4]   2> 1730244 T5969 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6018786168657202]
>>  [junit4]   2> 1730245 T5969 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@434ea2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4d5719cb),segFN=segments_1,generation=1}
>>  [junit4]   2> 1730245 T5969 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1730245 T5969 oass.SolrIndexSearcher.<init> Opening Searcher@682747bb main
>>  [junit4]   2> 1730246 T5954 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>>  [junit4]   2> 1730249 T5954 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>>  [junit4]   2> 1730249 T5995 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730251 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6018786168657202]
>>  [junit4]   2> 1730253 T5954 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@247327d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@392d8b48),segFN=segments_1,generation=1}
>>  [junit4]   2> 1730253 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1730254 T5954 oass.SolrIndexSearcher.<init> Opening Searcher@7487fd38 main
>>  [junit4]   2> 1730260 T6008 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@7487fd38 main{StandardDirectoryReader(segments_1:1:nrt)}
>>  [junit4]   2> 1730263 T5995 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730263 T5954 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
>>  [junit4]   2> 1730264 T5954 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:53698/od_jjq/cq collection:solrj_collection shard:shard1
>>  [junit4]   2> 1730264 T6007 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica2] Registered new searcher Searcher@682747bb main{StandardDirectoryReader(segments_1:1:nrt)}
>>  [junit4]   2> 1730268 T5954 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
>>  [junit4]   2> 1730288 T5995 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730288 T5969 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica2
>>  [junit4]   2> 1730289 T5969 oasc.ZkController.register Register replica - core:solrj_collection_shard2_replica2 address:http://127.0.0.1:53701/od_jjq/cq collection:solrj_collection shard:shard2
>>  [junit4]   2> 1730292 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730293 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730294 T5995 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730300 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730301 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730302 T5995 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1730302 T5995 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/
>>  [junit4]   2> 1730303 T5995 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
>>  [junit4]   2> 1730294 T5969 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard2/election
>>  [junit4]   2> 1730308 T5995 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data
>>  [junit4]   2> 1730309 T5995 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index/
>>  [junit4]   2> 1730310 T5995 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
>>  [junit4]   2> 1730311 T5995 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index
>>  [junit4]   2> 1730312 T5995 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
>>  [junit4]   2> 1730319 T5995 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>>  [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@858afaf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@659dfd2a),segFN=segments_1,generation=1}
>>  [junit4]   2> 1730320 T5995 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>>  [junit4]   2> 1730322 T5979 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730325 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
>>  [junit4]   2> 1730331 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>>  [junit4]   2> 1730332 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1730333 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>>  [junit4]   2> 1730333 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1730334 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>>  [junit4]   2> 1730335 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>>  [junit4]   2> 1730336 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1730338 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>>  [junit4]   2> 1730339 T5995 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>>  [junit4]   2> 1730339 T5979 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>>  [junit4]   2> 1730340 T5995 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>>  [junit4]   2> 1730341 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1730343 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1730343 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1730344 T5995 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>>  [junit4]   2> 1730344 T5995 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>>  [junit4]   2> 1730346 T5995 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>>  [junit4]   2> 1730346 T5954 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=-4
>>  [junit4]   2> 1730349 T5954 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
>>  [junit4]   2> 1730349 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>>  [junit4]   2> ASYNC  NEW_CORE C4181 name=solrj_collection_shard1_replica1 org.apache.solr.core.SolrCore@46d7249d url=http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1 node=127.0.0.1:53698_od_jjq%2Fcq C4181_STATE=coll:solrj_collection core:solrj_collection_shard1_replica1 props:{state=down, base_url=http://127.0.0.1:53698/od_jjq/cq, core=solrj_collection_shard1_replica1, node_name=127.0.0.1:53698_od_jjq%2Fcq}
>>  [junit4]   2> 1730350 T5954 C4181 P53698 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1/
>>  [junit4]   2> 1730350 T5954 C4181 P53698 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:53698/od_jjq/cq START replicas=[http://127.0.0.1:53707/od_jjq/cq/solrj_collection_shard1_replica2/] nUpdates=100
>>  [junit4]   2> 1730353 T5995 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>>  [junit4]   2> 1730354 T5995 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>>  [junit4]   2> 1730356 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>>  [junit4]   2> 1730354 T5979 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730358 T5995 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>>  [junit4]   2> 1730361 T5969 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
>>  [junit4]   2> 1730367 T5995 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>>  [junit4]   2> 1730369 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730372 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730373 T5979 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>>  [junit4]   2> 1730367 T5954 C4181 P53698 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:53698/od_jjq/cq DONE.  We have no versions.  sync failed.
>>  [junit4]   2> 1730374 T5954 C4181 P53698 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
>>  [junit4]   2> 1730374 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
>>  [junit4]   2> 1730379 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1730379 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1/ shard1
>>  [junit4]   2> 1730375 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>>  [junit4]   2> 1730384 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1730382 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1730381 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1730380 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1730385 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>>  [junit4]   2> 1730384 T5954 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leaders/shard1
>>  [junit4]   2> 1730391 T5969 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=-9
>>  [junit4]   2> 1730395 T5969 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
>>  [junit4]   2> 1730396 T5969 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>>  [junit4]   2> 1730396 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>>  [junit4]   2> 1730397 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>>  [junit4]   2> 1730397 T5979 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>>  [junit4]   2> 1730395 T5995 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>>  [junit4]   2> 1730399 T5995 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>>  [junit4]   2> 1730398 T5979 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/wo
>> 
>> [...truncated too long message...]
>> 
>> )
>>  [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:273)
>>  [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
>>  [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
>>  [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>>  [junit4]   2> 		at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>>  [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>>  [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>>  [junit4]   2> 	
>>  [junit4]   2> 2162194 T5987 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/nodes_used_collection/leader_elect/shard1/election/91100191848005644-core_node1-n_0000000001
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>>  [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:173)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:170)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:170)
>>  [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:67)
>>  [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:293)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>>  [junit4]   2> 
>>  [junit4]   2> 2162195 T5987 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>>  [junit4]   2> 2162195 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
>>  [junit4]   2> 2162195 T5987 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>>  [junit4]   2> 2162196 T5987 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>>  [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>>  [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>>  [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>>  [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>>  [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>>  [junit4]   2> 
>>  [junit4]   2> NOTE: test params are: codec=Appending, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_IQ, timezone=Etc/UTC
>>  [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.7.0_51 (64-bit)/cpus=2,threads=1,free=107685104,total=385253376
>>  [junit4]   2> NOTE: All tests run in this JVM: [PluginInfoTest, QueryElevationComponentTest, ZkCLITest, TestCollationField, CurrencyFieldXmlFileTest, BadCopyFieldTest, TestSolrXmlPersistence, TestPartialUpdateDeduplication, TestManagedSchema, WordBreakSolrSpellCheckerTest, RAMDirectoryFactoryTest, TestDistributedMissingSort, TestJmxIntegration, TestElisionMultitermQuery, SpellCheckComponentTest, ZkSolrClientTest, HdfsBasicDistributedZk2Test, TestArbitraryIndexDir, TestSimpleQParserPlugin, TestRandomDVFaceting, TestSchemaSimilarityResource, OutputWriterTest, TestQueryTypes, CopyFieldTest, TestStressReorder, DistanceFunctionTest, SuggestComponentTest, MigrateRouteKeyTest, DateFieldTest, DistributedDebugComponentTest, TestCloudManagedSchemaAddField, FieldAnalysisRequestHandlerTest, TestFileDictionaryLookup, TestDocBasedVersionConstraints, LeaderElectionIntegrationTest, TestCollapseQParserPlugin, TestMultiCoreConfBootstrap, QueryResultKeyTest, TestDocSet, TestLMDirichletSimilarityFactory, TestCSVLoader, TestRTGBase, TestSchemaVersionResource, FileUtilsTest, TriLevelCompositeIdRoutingTest, ParsingFieldUpdateProcessorsTest, TestCodecSupport, DistributedTermsComponentTest, TestCustomSort, ConnectionManagerTest, RequiredFieldsTest, TestQueryUtils, TestReplicationHandler, TestSolrXMLSerializer, TestUtils, UpdateRequestProcessorFactoryTest, TestFastWriter, TestPHPSerializedResponseWriter, OpenExchangeRatesOrgProviderTest, TestFoldingMultitermQuery, TestNonNRTOpen, DOMUtilTest, CSVRequestHandlerTest, TestIndexingPerformance, DocumentAnalysisRequestHandlerTest, TestLuceneMatchVersion, DocValuesTest, TestJoin, TestSchemaNameResource, TestBadConfig, CoreAdminCreateDiscoverTest, HighlighterConfigTest, TestPerFieldSimilarity, BlockCacheTest, SolrIndexSplitterTest, URLClassifyProcessorTest, SchemaVersionSpecificBehaviorTest, QueryParsingTest, CursorMarkTest, SimplePostToolTest, TestNumberUtils, EnumFieldTest, ChaosMonkeySafeLeaderTest, DirectUpdateHandlerOptimizeTest, TestCollationFieldDocValues, TestFuzzyAnalyzedSuggestions, TestQuerySenderListener, HdfsRecoveryZkTest, CoreMergeIndexesAdminHandlerTest, ClusterStateTest, TestSolr4Spatial, TestSystemIdResolver, TestStressLucene, SolrCoreCheckLockOnStartupTest, TestManagedSchemaFieldResource, IndexSchemaTest, TestWordDelimiterFilterFactory, TestOmitPositions, TermVectorComponentTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestIndexSearcher, TestPseudoReturnFields, HdfsChaosMonkeySafeLeaderTest, TestLazyCores, DateMathParserTest, TestAtomicUpdateErrorCases, TestLFUCache, TestDefaultSimilarityFactory, TestPostingsSolrHighlighter, SimpleFacetsTest, TestImplicitCoreProperties, DeleteReplicaTest, TestSerializedLuceneMatchVersion, TestHighFrequencyDictionaryFactory, ChangedSchemaMergeTest, TestAnalyzedSuggestions, CursorPagingTest, PingRequestHandlerTest, TestSuggestSpellingConverter, TestDFRSimilarityFactory, TestSolrDeletionPolicy2, PrimUtilsTest, SignatureUpdateProcessorFactoryTest, TestStressUserVersions, TestDistribDocBasedVersion, PrimitiveFieldTypeTest, ShardRoutingCustomTest, TestCloudManagedSchema, TestDynamicFieldCollectionResource, AutoCommitTest, SolrPluginUtilsTest, BlockDirectoryTest, SolrIndexConfigTest, ShowFileRequestHandlerTest, JSONWriterTest, TestStressVersions, SpatialFilterTest, BadIndexSchemaTest, ShardRoutingTest, TestShardHandlerFactory, SuggesterTest, TestBM25SimilarityFactory, DirectUpdateHandlerTest, DistribCursorPagingTest, FieldMutatingUpdateProcessorTest, RecoveryZkTest, TestNoOpRegenerator, TestSolrDeletionPolicy1, BasicDistributedZkTest, TestBinaryResponseWriter, TestRandomMergePolicy, ExternalFileFieldSortTest, OpenCloseCoreStressTest, PolyFieldTest, BasicDistributedZk2Test, TestFreeTextSuggestions, JsonLoaderTest, TestAddFieldRealTimeGet, UniqFieldsUpdateProcessorFactoryTest, TestAnalyzeInfixSuggestions, TestStressRecovery, TestBinaryField, IndexSchemaRuntimeFieldTest, TestSolrQueryParser, HdfsBasicDistributedZkTest, ZkNodePropsTest, BadComponentTest, UUIDFieldTest, AssignTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
>>  [junit4] Completed in 443.02s, 1 test, 1 failure <<< FAILURES!
>> 
>> [...truncated 539 lines...]
>> BUILD FAILED
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:459: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:439: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/extra-targets.xml:37: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:491: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1308: The following error occurred while executing this line:
>> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:941: There were test failures: 354 suites, 1540 tests, 1 failure, 40 ignored (6 assumptions)
>> 
>> Total time: 85 minutes 49 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
>> Archiving artifacts
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>> 
>> 
>> 
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
> 


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-4.x-MacOSX (64bit/jdk1.7.0) - Build # 1210 - Still Failing!

Posted by Mark Miller <ma...@gmail.com>.
Looks like it’s stuck in a socket read. I’ll turn the tests distributed so timeout back on.

- Mark

On Jan 18, 2014, at 4:22 PM, Policeman Jenkins Server <je...@thetaphi.de> wrote:

> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-MacOSX/1210/
> Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
> 
> 1 tests failed.
> FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch
> 
> Error Message:
> There are still nodes recoverying - waited for 330 seconds
> 
> Stack Trace:
> java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
> 	at __randomizedtesting.SeedInfo.seed([E01973DE033FE27B:61FFFDC674608247]:0)
> 	at org.junit.Assert.fail(Assert.java:93)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:172)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:131)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:126)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testCollectionsAPI(CollectionsAPIDistributedZkTest.java:772)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:196)
> 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:867)
> 	at sun.reflect.GeneratedMethodAccessor52.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:606)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
> 	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
> 	at java.lang.Thread.run(Thread.java:744)
> 
> 
> 
> 
> Build Log:
> [...truncated 10429 lines...]
>   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
>   [junit4]   2> 1719213 T5915 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /od_jjq/cq
>   [junit4]   2> 1719222 T5915 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
>   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./solrtest-CollectionsAPIDistributedZkTest-1390078610952
>   [junit4]   2> 1719224 T5915 oasc.ZkTestServer.run STARTING ZK TEST SERVER
>   [junit4]   2> 1719225 T5916 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
>   [junit4]   2> 1719327 T5915 oasc.ZkTestServer.run start zk server on port:53689
>   [junit4]   2> 1719329 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1719338 T5922 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b2bf8c1 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1719338 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1719339 T5915 oascc.SolrZkClient.makePath makePath: /solr
>   [junit4]   2> 1719353 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1719358 T5924 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31e8fc2a name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1719359 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1719359 T5915 oascc.SolrZkClient.makePath makePath: /collections/collection1
>   [junit4]   2> 1719375 T5915 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
>   [junit4]   2> 1719384 T5915 oascc.SolrZkClient.makePath makePath: /collections/control_collection
>   [junit4]   2> 1719393 T5915 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
>   [junit4]   2> 1719403 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>   [junit4]   2> 1719404 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
>   [junit4]   2> 1719421 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
>   [junit4]   2> 1719422 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
>   [junit4]   2> 1719432 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1719433 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1719444 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>   [junit4]   2> 1719445 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
>   [junit4]   2> 1719456 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>   [junit4]   2> 1719457 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
>   [junit4]   2> 1719469 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>   [junit4]   2> 1719470 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
>   [junit4]   2> 1719480 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>   [junit4]   2> 1719481 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
>   [junit4]   2> 1719492 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
>   [junit4]   2> 1719493 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
>   [junit4]   2> 1719503 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1719504 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1719513 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
>   [junit4]   2> 1719514 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
>   [junit4]   2> 1719524 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
>   [junit4]   2> 1719526 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
>   [junit4]   2> Using old style solr.xml
>   [junit4]   2> 1719542 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1719547 T5926 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66930f41 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1719547 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1719548 T5915 oascc.SolrZkClient.makePath makePath: /solr
>   [junit4]   2> 1719558 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1719562 T5928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3239c4d9 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1719562 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1719564 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
>   [junit4]   2> 1719565 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
>   [junit4]   2> 1719576 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
>   [junit4]   2> 1719578 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
>   [junit4]   2> 1719588 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1719589 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 1719600 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
>   [junit4]   2> 1719600 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
>   [junit4]   2> 1719611 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
>   [junit4]   2> 1719612 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
>   [junit4]   2> 1719622 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
>   [junit4]   2> 1719624 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
>   [junit4]   2> 1719633 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
>   [junit4]   2> 1719634 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
>   [junit4]   2> 1719650 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1719651 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
>   [junit4]   2> 1719683 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
>   [junit4]   2> 1719684 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
>   [junit4]   2> 1719694 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
>   [junit4]   2> 1719695 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
>   [junit4]   2> 1719706 T5915 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
>   [junit4]   2> 1719707 T5915 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
>   [junit4]   2> 1720156 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1720161 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53694
>   [junit4]   2> 1720162 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1720163 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1720164 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458
>   [junit4]   2> 1720164 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/'
>   [junit4]   2> 1720208 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/solr.xml
>   [junit4]   2> 1720327 T5915 oasc.CoreContainer.<init> New CoreContainer 370136241
>   [junit4]   2> 1720328 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-controljetty-1390078611458/]
>   [junit4]   2> 1720330 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1720330 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1720331 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1720332 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1720333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1720333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1720334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1720334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1720335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1720350 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1720351 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1720352 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1720352 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>   [junit4]   2> 1720353 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1720355 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1720360 T5939 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6954bbc7 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1720361 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1720370 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1720374 T5941 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f18b9ff name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1720375 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1720381 T5915 oascc.SolrZkClient.makePath makePath: /overseer/queue
>   [junit4]   2> 1720394 T5915 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
>   [junit4]   2> 1720406 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes
>   [junit4]   2> 1720412 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53694_od_jjq%2Fcq
>   [junit4]   2> 1720416 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53694_od_jjq%2Fcq
>   [junit4]   2> 1720428 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect
>   [junit4]   2> 1720436 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
>   [junit4]   2> 1720451 T5915 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
>   [junit4]   2> 1720460 T5915 oasc.Overseer.start Overseer (id=91100191848005637-127.0.0.1:53694_od_jjq%2Fcq-n_0000000000) starting
>   [junit4]   2> 1720471 T5915 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
>   [junit4]   2> 1720486 T5943 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
>   [junit4]   2> 1720488 T5915 oascc.SolrZkClient.makePath makePath: /clusterstate.json
>   [junit4]   2> 1720490 T5943 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
>   [junit4]   2> 1720497 T5915 oascc.SolrZkClient.makePath makePath: /aliases.json
>   [junit4]   2> 1720504 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1720515 T5942 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
>   [junit4]   2> 1720523 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1720524 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1720527 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1720532 T5946 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4fdc9238 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1720532 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1720536 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1720546 T5915 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
>   [junit4]   2> 1721004 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1721012 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53698
>   [junit4]   2> 1721013 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1721013 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1721014 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275
>   [junit4]   2> 1721014 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/'
>   [junit4]   2> 1721059 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solr.xml
>   [junit4]   2> 1721165 T5915 oasc.CoreContainer.<init> New CoreContainer 2108103389
>   [junit4]   2> 1721165 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/]
>   [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1721167 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1721168 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1721169 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1721186 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1721187 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1721187 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1721188 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>   [junit4]   2> 1721189 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1721192 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1721207 T5957 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@65d4e39d name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1721207 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1721220 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1721228 T5959 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4db4e5c6 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1721229 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1721243 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1722255 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53698_od_jjq%2Fcq
>   [junit4]   2> 1722258 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53698_od_jjq%2Fcq
>   [junit4]   2> 1722266 T5959 oascc.ZkStateReader$3.process Updating live nodes... (2)
>   [junit4]   2> 1722266 T5941 oascc.ZkStateReader$3.process Updating live nodes... (2)
>   [junit4]   2> 1722267 T5946 oascc.ZkStateReader$3.process Updating live nodes... (2)
>   [junit4]   2> 1722281 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1722281 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1722625 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1722630 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53701
>   [junit4]   2> 1722631 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011
>   [junit4]   2> 1722632 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/'
>   [junit4]   2> 1722668 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solr.xml
>   [junit4]   2> 1722738 T5915 oasc.CoreContainer.<init> New CoreContainer 1501927636
>   [junit4]   2> 1722739 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/]
>   [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1722741 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1722742 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1722743 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1722743 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1722753 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1722753 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1722753 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1722754 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>   [junit4]   2> 1722754 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1722755 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1722760 T5971 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@51f4b436 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1722760 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1722767 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1722771 T5973 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b11c2d8 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1722771 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1722782 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1723792 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53701_od_jjq%2Fcq
>   [junit4]   2> 1723795 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53701_od_jjq%2Fcq
>   [junit4]   2> 1723805 T5941 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1723806 T5946 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1723807 T5959 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1723807 T5973 oascc.ZkStateReader$3.process Updating live nodes... (3)
>   [junit4]   2> 1723827 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1723828 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1724210 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1724215 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53704
>   [junit4]   2> 1724216 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1724216 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1724217 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557
>   [junit4]   2> 1724217 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/'
>   [junit4]   2> 1724246 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solr.xml
>   [junit4]   2> 1724331 T5915 oasc.CoreContainer.<init> New CoreContainer 1311775510
>   [junit4]   2> 1724332 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/]
>   [junit4]   2> 1724333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1724333 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1724334 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1724335 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1724346 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1724347 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1724347 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1724347 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>   [junit4]   2> 1724348 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1724349 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1724354 T5985 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38426974 name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1724354 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1724361 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1724364 T5987 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22116710 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1724365 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1724377 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1725388 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53704_od_jjq%2Fcq
>   [junit4]   2> 1725391 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53704_od_jjq%2Fcq
>   [junit4]   2> 1725402 T5946 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1725402 T5941 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1725403 T5959 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1725403 T5987 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1725403 T5973 oascc.ZkStateReader$3.process Updating live nodes... (4)
>   [junit4]   2> 1725420 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1725421 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1725880 T5915 oejs.Server.doStart jetty-8.1.10.v20130312
>   [junit4]   2> 1725885 T5915 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53707
>   [junit4]   2> 1725886 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
>   [junit4]   2> 1725886 T5915 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
>   [junit4]   2> 1725887 T5915 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150
>   [junit4]   2> 1725888 T5915 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/'
>   [junit4]   2> 1725922 T5915 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solr.xml
>   [junit4]   2> 1725993 T5915 oasc.CoreContainer.<init> New CoreContainer 1201695941
>   [junit4]   2> 1725994 T5915 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/]
>   [junit4]   2> 1725995 T5915 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
>   [junit4]   2> 1725995 T5915 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
>   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
>   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
>   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
>   [junit4]   2> 1725996 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
>   [junit4]   2> 1725997 T5915 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
>   [junit4]   2> 1725997 T5915 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
>   [junit4]   2> 1725998 T5915 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
>   [junit4]   2> 1726008 T5915 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
>   [junit4]   2> 1726008 T5915 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
>   [junit4]   2> 1726008 T5915 oasc.CoreContainer.load Host Name: 127.0.0.1
>   [junit4]   2> 1726009 T5915 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:53689/solr
>   [junit4]   2> 1726009 T5915 oasc.ZkController.checkChrootPath zkHost includes chroot
>   [junit4]   2> 1726010 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1726014 T5999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75451e3f name:ZooKeeperConnection Watcher:127.0.0.1:53689 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1726015 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1726022 T5915 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
>   [junit4]   2> 1726028 T6001 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@509b80a4 name:ZooKeeperConnection Watcher:127.0.0.1:53689/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 1726028 T5915 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
>   [junit4]   2> 1726040 T5915 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
>   [junit4]   2> 1727054 T5915 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53707_od_jjq%2Fcq
>   [junit4]   2> 1727056 T5915 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53707_od_jjq%2Fcq
>   [junit4]   2> 1727068 T5959 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1727068 T5946 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1727068 T5987 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1727068 T6001 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1727068 T5941 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1727068 T5973 oascc.ZkStateReader$3.process Updating live nodes... (5)
>   [junit4]   2> 1727088 T5915 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0
>   [junit4]   2> 1727088 T5915 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
>   [junit4]   2> 1727135 T5953 oasha.CollectionsHandler.handleCreateAction Creating Collection : version=2&numShards=2&action=CREATE&collection.configName=conf1&router.field=myOwnField&name=solrj_collection&replicationFactor=2&wt=javabin&router.name=compositeId
>   [junit4]   2> 1727138 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727142 T5943 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
>   [junit4]   2> 	  "operation":"createcollection",
>   [junit4]   2> 	  "fromApi":"true",
>   [junit4]   2> 	  "name":"solrj_collection",
>   [junit4]   2> 	  "replicationFactor":"2",
>   [junit4]   2> 	  "collection.configName":"conf1",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "router.field":"myOwnField",
>   [junit4]   2> 	  "router.name":"compositeId"}
>   [junit4]   2> 1727142 T5943 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
>   [junit4]   2> 	  "operation":"createcollection",
>   [junit4]   2> 	  "fromApi":"true",
>   [junit4]   2> 	  "name":"solrj_collection",
>   [junit4]   2> 	  "replicationFactor":"2",
>   [junit4]   2> 	  "collection.configName":"conf1",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "router.field":"myOwnField",
>   [junit4]   2> 	  "router.name":"compositeId"}
>   [junit4]   2> 1727143 T5943 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
>   [junit4]   2> 1727143 T5943 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
>   [junit4]   2> 1727156 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727160 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>   [junit4]   2> 1727161 T5942 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
>   [junit4]   2> 1727162 T5942 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
>   [junit4]   2> 1727169 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727172 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727172 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727172 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727172 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727172 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727172 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727257 T5943 oasc.OverseerCollectionProcessor.createCollection going to create cores replicas shardNames [shard1, shard2] , repFactor : 2
>   [junit4]   2> 1727258 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:53698_od_jjq%2Fcq
>   [junit4]   2> 1727259 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:53707_od_jjq%2Fcq
>   [junit4]   2> 1727260 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:53704_od_jjq%2Fcq
>   [junit4]   2> 1727262 T5943 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:53701_od_jjq%2Fcq
>   [junit4]   2> 1727267 T5954 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down
>   [junit4]   2> 1727271 T5979 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down
>   [junit4]   2> 1727273 T5995 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down
>   [junit4]   2> 1727273 T5969 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down
>   [junit4]   2> 1727276 T5954 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1727277 T5979 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1727277 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727279 T5969 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1727281 T5995 oasc.ZkController.waitForCoreNodeName look for our core node name
>   [junit4]   2> 1727284 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>   [junit4]   2> 1727287 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:53698/od_jjq/cq",
>   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:53698_od_jjq%2Fcq",
>   [junit4]   2> 	  "shard":"shard1",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1727295 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727300 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:53704/od_jjq/cq",
>   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:53704_od_jjq%2Fcq",
>   [junit4]   2> 	  "shard":"shard2",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1727310 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727314 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:53701/od_jjq/cq",
>   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:53701_od_jjq%2Fcq",
>   [junit4]   2> 	  "shard":"shard2",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1727321 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727324 T5942 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
>   [junit4]   2> 	  "operation":"state",
>   [junit4]   2> 	  "state":"down",
>   [junit4]   2> 	  "base_url":"http://127.0.0.1:53707/od_jjq/cq",
>   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
>   [junit4]   2> 	  "roles":null,
>   [junit4]   2> 	  "node_name":"127.0.0.1:53707_od_jjq%2Fcq",
>   [junit4]   2> 	  "shard":"shard1",
>   [junit4]   2> 	  "shard_range":null,
>   [junit4]   2> 	  "shard_state":"active",
>   [junit4]   2> 	  "shard_parent":null,
>   [junit4]   2> 	  "collection":"solrj_collection",
>   [junit4]   2> 	  "numShards":"2",
>   [junit4]   2> 	  "core_node_name":null}
>   [junit4]   2> 1727332 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1727445 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727445 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727446 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727445 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727446 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1727446 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1728278 T5954 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
>   [junit4]   2> 1728278 T5979 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
>   [junit4]   2> 1728279 T5954 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solr.xml
>   [junit4]   2> 1728279 T5979 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solr.xml
>   [junit4]   2> 1728280 T5979 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solrj_collection_shard2_replica1
>   [junit4]   2> 1728280 T5979 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1728281 T5969 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
>   [junit4]   2> 1728280 T5954 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1
>   [junit4]   2> 1728281 T5954 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1728282 T5969 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solr.xml
>   [junit4]   2> 1728282 T5969 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2
>   [junit4]   2> 1728282 T5969 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1728282 T5995 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
>   [junit4]   2> 1728284 T5995 oasc.SolrXMLCoresLocator.doPersist Persisted core descriptions to /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solr.xml
>   [junit4]   2> 1728284 T5979 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1728284 T5995 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using instanceDir: ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2
>   [junit4]   2> 1728284 T5954 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1728285 T5954 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1728285 T5995 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
>   [junit4]   2> 1728285 T5969 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1728286 T5969 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1728284 T5979 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1728287 T5995 oasc.ZkController.createCollectionZkNode Collection zkNode exists
>   [junit4]   2> 1728287 T5995 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
>   [junit4]   2> 1728289 T5954 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/'
>   [junit4]   2> 1728291 T5995 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/'
>   [junit4]   2> 1728292 T5969 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/'
>   [junit4]   2> 1728295 T5979 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty3-1390078615557/solrj_collection_shard2_replica1/'
>   [junit4]   2> 1728399 T5954 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1728413 T5995 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1728426 T5969 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1728492 T5979 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
>   [junit4]   2> 1728547 T5969 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1728550 T5995 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1728554 T5969 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1728574 T5995 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1728577 T5969 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
>   [junit4]   2> 1728610 T5995 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
>   [junit4]   2> 1728640 T5954 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1728652 T5979 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
>   [junit4]   2> 1728653 T5954 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1728663 T5954 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
>   [junit4]   2> 1728663 T5979 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
>   [junit4]   2> 1728715 T5979 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
>   [junit4]   2> 1730003 T5954 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730003 T5954 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730004 T5954 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1730012 T5954 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1730018 T5954 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1730071 T5969 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730071 T5969 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730073 T5969 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1730090 T5969 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1730097 T5969 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1730128 T5995 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730128 T5995 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730129 T5995 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1730129 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730137 T5969 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730141 T5954 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730153 T5995 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1730158 T5995 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1730166 T5969 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730176 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730180 T5969 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730181 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730182 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730182 T5954 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730182 T5979 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730184 T5979 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
>   [junit4]   2> 1730184 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730184 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730185 T5969 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730186 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730186 T5954 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730187 T5954 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1730187 T5954 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/, dataDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/
>   [junit4]   2> 1730188 T5954 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
>   [junit4]   2> 1730185 T5979 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
>   [junit4]   2> 1730189 T5954 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data
>   [junit4]   2> 1730189 T5954 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index/
>   [junit4]   2> 1730190 T5954 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
>   [junit4]   2> 1730190 T5954 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty1-1390078612275/solrj_collection_shard1_replica1/data/index
>   [junit4]   2> 1730191 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
>   [junit4]   2> 1730192 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730192 T5969 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730192 T5969 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1730193 T5969 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/, dataDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/
>   [junit4]   2> 1730193 T5969 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
>   [junit4]   2> 1730194 T5969 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data
>   [junit4]   2> 1730195 T5969 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index/
>   [junit4]   2> 1730195 T5969 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
>   [junit4]   2> 1730197 T5969 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty2-1390078614011/solrj_collection_shard2_replica2/data/index
>   [junit4]   2> 1730197 T5969 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
>   [junit4]   2> 1730199 T5954 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@247327d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@392d8b48),segFN=segments_1,generation=1}
>   [junit4]   2> 1730199 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1730199 T5969 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@434ea2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4d5719cb),segFN=segments_1,generation=1}
>   [junit4]   2> 1730200 T5969 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1730200 T5979 oass.IndexSchema.readSchema default search field in schema is text
>   [junit4]   2> 1730204 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>   [junit4]   2> 1730204 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1730205 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1730206 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1730207 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1730207 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>   [junit4]   2> 1730208 T5969 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1730209 T5969 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1730209 T5969 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>   [junit4]   2> 1730204 T5979 oass.IndexSchema.readSchema unique key field: id
>   [junit4]   2> 1730210 T5969 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>   [junit4]   2> 1730212 T5969 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>   [junit4]   2> 1730213 T5969 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>   [junit4]   2> 1730213 T5969 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>   [junit4]   2> 1730214 T5969 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>   [junit4]   2> 1730215 T5969 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>   [junit4]   2> 1730209 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>   [junit4]   2> 1730216 T5969 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>   [junit4]   2> 1730216 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1730217 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1730217 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1730218 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1730219 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>   [junit4]   2> 1730220 T5954 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1730220 T5954 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1730221 T5954 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>   [junit4]   2> 1730222 T5954 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>   [junit4]   2> 1730222 T5954 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>   [junit4]   2> 1730223 T5954 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>   [junit4]   2> 1730223 T5954 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>   [junit4]   2> 1730224 T5954 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>   [junit4]   2> 1730227 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>   [junit4]   2> 1730226 T5969 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>   [junit4]   2> 1730228 T5954 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>   [junit4]   2> 1730235 T5954 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>   [junit4]   2> 1730240 T5969 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>   [junit4]   2> 1730241 T5969 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>   [junit4]   2> 1730244 T5969 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6018786168657202]
>   [junit4]   2> 1730245 T5969 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@434ea2f6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4d5719cb),segFN=segments_1,generation=1}
>   [junit4]   2> 1730245 T5969 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1730245 T5969 oass.SolrIndexSearcher.<init> Opening Searcher@682747bb main
>   [junit4]   2> 1730246 T5954 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>   [junit4]   2> 1730249 T5954 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>   [junit4]   2> 1730249 T5995 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730251 T5954 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=43, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6018786168657202]
>   [junit4]   2> 1730253 T5954 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@247327d5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@392d8b48),segFN=segments_1,generation=1}
>   [junit4]   2> 1730253 T5954 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1730254 T5954 oass.SolrIndexSearcher.<init> Opening Searcher@7487fd38 main
>   [junit4]   2> 1730260 T6008 oasc.SolrCore.registerSearcher [solrj_collection_shard1_replica1] Registered new searcher Searcher@7487fd38 main{StandardDirectoryReader(segments_1:1:nrt)}
>   [junit4]   2> 1730263 T5995 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730263 T5954 oasc.CoreContainer.registerCore registering core: solrj_collection_shard1_replica1
>   [junit4]   2> 1730264 T5954 oasc.ZkController.register Register replica - core:solrj_collection_shard1_replica1 address:http://127.0.0.1:53698/od_jjq/cq collection:solrj_collection shard:shard1
>   [junit4]   2> 1730264 T6007 oasc.SolrCore.registerSearcher [solrj_collection_shard2_replica2] Registered new searcher Searcher@682747bb main{StandardDirectoryReader(segments_1:1:nrt)}
>   [junit4]   2> 1730268 T5954 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard1/election
>   [junit4]   2> 1730288 T5995 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730288 T5969 oasc.CoreContainer.registerCore registering core: solrj_collection_shard2_replica2
>   [junit4]   2> 1730289 T5969 oasc.ZkController.register Register replica - core:solrj_collection_shard2_replica2 address:http://127.0.0.1:53701/od_jjq/cq collection:solrj_collection shard:shard2
>   [junit4]   2> 1730292 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730293 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730294 T5995 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730300 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730301 T5995 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730302 T5995 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1730302 T5995 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/, dataDir=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/
>   [junit4]   2> 1730303 T5995 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@589e0cdc
>   [junit4]   2> 1730294 T5969 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leader_elect/shard2/election
>   [junit4]   2> 1730308 T5995 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data
>   [junit4]   2> 1730309 T5995 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index/
>   [junit4]   2> 1730310 T5995 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory './org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
>   [junit4]   2> 1730311 T5995 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.CollectionsAPIDistributedZkTest-jetty4-1390078617150/solrj_collection_shard1_replica2/data/index
>   [junit4]   2> 1730312 T5995 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=25, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
>   [junit4]   2> 1730319 T5995 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
>   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@858afaf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@659dfd2a),segFN=segments_1,generation=1}
>   [junit4]   2> 1730320 T5995 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
>   [junit4]   2> 1730322 T5979 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730325 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
>   [junit4]   2> 1730331 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
>   [junit4]   2> 1730332 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1730333 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
>   [junit4]   2> 1730333 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1730334 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
>   [junit4]   2> 1730335 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
>   [junit4]   2> 1730336 T5995 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1730338 T5995 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
>   [junit4]   2> 1730339 T5995 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
>   [junit4]   2> 1730339 T5979 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
>   [junit4]   2> 1730340 T5995 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
>   [junit4]   2> 1730341 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1730343 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1730343 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1730344 T5995 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
>   [junit4]   2> 1730344 T5995 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
>   [junit4]   2> 1730346 T5995 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
>   [junit4]   2> 1730346 T5954 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard1: total=2 found=1 timeoutin=-4
>   [junit4]   2> 1730349 T5954 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
>   [junit4]   2> 1730349 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>   [junit4]   2> ASYNC  NEW_CORE C4181 name=solrj_collection_shard1_replica1 org.apache.solr.core.SolrCore@46d7249d url=http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1 node=127.0.0.1:53698_od_jjq%2Fcq C4181_STATE=coll:solrj_collection core:solrj_collection_shard1_replica1 props:{state=down, base_url=http://127.0.0.1:53698/od_jjq/cq, core=solrj_collection_shard1_replica1, node_name=127.0.0.1:53698_od_jjq%2Fcq}
>   [junit4]   2> 1730350 T5954 C4181 P53698 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1/
>   [junit4]   2> 1730350 T5954 C4181 P53698 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:53698/od_jjq/cq START replicas=[http://127.0.0.1:53707/od_jjq/cq/solrj_collection_shard1_replica2/] nUpdates=100
>   [junit4]   2> 1730353 T5995 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
>   [junit4]   2> 1730354 T5995 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
>   [junit4]   2> 1730356 T5942 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
>   [junit4]   2> 1730354 T5979 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730358 T5995 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
>   [junit4]   2> 1730361 T5969 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
>   [junit4]   2> 1730367 T5995 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
>   [junit4]   2> 1730369 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730372 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730373 T5979 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
>   [junit4]   2> 1730367 T5954 C4181 P53698 oasu.PeerSync.sync PeerSync: core=solrj_collection_shard1_replica1 url=http://127.0.0.1:53698/od_jjq/cq DONE.  We have no versions.  sync failed.
>   [junit4]   2> 1730374 T5954 C4181 P53698 oasc.SyncStrategy.syncReplicas Leader's attempt to sync with shard failed, moving to the next candidate
>   [junit4]   2> 1730374 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
>   [junit4]   2> 1730379 T5946 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1730379 T5954 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53698/od_jjq/cq/solrj_collection_shard1_replica1/ shard1
>   [junit4]   2> 1730375 T5941 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
>   [junit4]   2> 1730384 T5941 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1730382 T6001 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1730381 T5973 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1730380 T5959 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1730385 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
>   [junit4]   2> 1730384 T5954 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection/leaders/shard1
>   [junit4]   2> 1730391 T5969 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Waiting until we see more replicas up for shard shard2: total=2 found=1 timeoutin=-9
>   [junit4]   2> 1730395 T5969 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Was waiting for replicas to come up, but they are taking too long - assuming they won't come back till later
>   [junit4]   2> 1730396 T5969 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
>   [junit4]   2> 1730396 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
>   [junit4]   2> 1730397 T5979 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
>   [junit4]   2> 1730397 T5979 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
>   [junit4]   2> 1730395 T5995 oasu.CommitTracker.<init> Hard AutoCommit: disabled
>   [junit4]   2> 1730399 T5995 oasu.CommitTracker.<init> Soft AutoCommit: disabled
>   [junit4]   2> 1730398 T5979 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/wo
> 
> [...truncated too long message...]
> 
> )
>   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:273)
>   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
>   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
>   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>   [junit4]   2> 	
>   [junit4]   2> 2162194 T5987 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/nodes_used_collection/leader_elect/shard1/election/91100191848005644-core_node1-n_0000000001
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.delete(ZooKeeper.java:873)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:173)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$2.execute(SolrZkClient.java:170)
>   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.delete(SolrZkClient.java:170)
>   [junit4]   2> 	at org.apache.solr.cloud.ElectionContext.cancelElection(ElectionContext.java:67)
>   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:293)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:164)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:108)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>   [junit4]   2> 
>   [junit4]   2> 2162195 T5987 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>   [junit4]   2> 2162195 T5987 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
>   [junit4]   2> 2162195 T5987 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
>   [junit4]   2> 2162196 T5987 oasc.LeaderElector$1.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
>   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
>   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1468)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:256)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:253)
>   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
>   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:253)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:92)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$000(LeaderElector.java:55)
>   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$1.process(LeaderElector.java:137)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:519)
>   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:495)
>   [junit4]   2> 
>   [junit4]   2> NOTE: test params are: codec=Appending, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=ar_IQ, timezone=Etc/UTC
>   [junit4]   2> NOTE: Mac OS X 10.8.5 x86_64/Oracle Corporation 1.7.0_51 (64-bit)/cpus=2,threads=1,free=107685104,total=385253376
>   [junit4]   2> NOTE: All tests run in this JVM: [PluginInfoTest, QueryElevationComponentTest, ZkCLITest, TestCollationField, CurrencyFieldXmlFileTest, BadCopyFieldTest, TestSolrXmlPersistence, TestPartialUpdateDeduplication, TestManagedSchema, WordBreakSolrSpellCheckerTest, RAMDirectoryFactoryTest, TestDistributedMissingSort, TestJmxIntegration, TestElisionMultitermQuery, SpellCheckComponentTest, ZkSolrClientTest, HdfsBasicDistributedZk2Test, TestArbitraryIndexDir, TestSimpleQParserPlugin, TestRandomDVFaceting, TestSchemaSimilarityResource, OutputWriterTest, TestQueryTypes, CopyFieldTest, TestStressReorder, DistanceFunctionTest, SuggestComponentTest, MigrateRouteKeyTest, DateFieldTest, DistributedDebugComponentTest, TestCloudManagedSchemaAddField, FieldAnalysisRequestHandlerTest, TestFileDictionaryLookup, TestDocBasedVersionConstraints, LeaderElectionIntegrationTest, TestCollapseQParserPlugin, TestMultiCoreConfBootstrap, QueryResultKeyTest, TestDocSet, TestLMDirichletSimilarityFactory, TestCSVLoader, TestRTGBase, TestSchemaVersionResource, FileUtilsTest, TriLevelCompositeIdRoutingTest, ParsingFieldUpdateProcessorsTest, TestCodecSupport, DistributedTermsComponentTest, TestCustomSort, ConnectionManagerTest, RequiredFieldsTest, TestQueryUtils, TestReplicationHandler, TestSolrXMLSerializer, TestUtils, UpdateRequestProcessorFactoryTest, TestFastWriter, TestPHPSerializedResponseWriter, OpenExchangeRatesOrgProviderTest, TestFoldingMultitermQuery, TestNonNRTOpen, DOMUtilTest, CSVRequestHandlerTest, TestIndexingPerformance, DocumentAnalysisRequestHandlerTest, TestLuceneMatchVersion, DocValuesTest, TestJoin, TestSchemaNameResource, TestBadConfig, CoreAdminCreateDiscoverTest, HighlighterConfigTest, TestPerFieldSimilarity, BlockCacheTest, SolrIndexSplitterTest, URLClassifyProcessorTest, SchemaVersionSpecificBehaviorTest, QueryParsingTest, CursorMarkTest, SimplePostToolTest, TestNumberUtils, EnumFieldTest, ChaosMonkeySafeLeaderTest, DirectUpdateHandlerOptimizeTest, TestCollationFieldDocValues, TestFuzzyAnalyzedSuggestions, TestQuerySenderListener, HdfsRecoveryZkTest, CoreMergeIndexesAdminHandlerTest, ClusterStateTest, TestSolr4Spatial, TestSystemIdResolver, TestStressLucene, SolrCoreCheckLockOnStartupTest, TestManagedSchemaFieldResource, IndexSchemaTest, TestWordDelimiterFilterFactory, TestOmitPositions, TermVectorComponentTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestIndexSearcher, TestPseudoReturnFields, HdfsChaosMonkeySafeLeaderTest, TestLazyCores, DateMathParserTest, TestAtomicUpdateErrorCases, TestLFUCache, TestDefaultSimilarityFactory, TestPostingsSolrHighlighter, SimpleFacetsTest, TestImplicitCoreProperties, DeleteReplicaTest, TestSerializedLuceneMatchVersion, TestHighFrequencyDictionaryFactory, ChangedSchemaMergeTest, TestAnalyzedSuggestions, CursorPagingTest, PingRequestHandlerTest, TestSuggestSpellingConverter, TestDFRSimilarityFactory, TestSolrDeletionPolicy2, PrimUtilsTest, SignatureUpdateProcessorFactoryTest, TestStressUserVersions, TestDistribDocBasedVersion, PrimitiveFieldTypeTest, ShardRoutingCustomTest, TestCloudManagedSchema, TestDynamicFieldCollectionResource, AutoCommitTest, SolrPluginUtilsTest, BlockDirectoryTest, SolrIndexConfigTest, ShowFileRequestHandlerTest, JSONWriterTest, TestStressVersions, SpatialFilterTest, BadIndexSchemaTest, ShardRoutingTest, TestShardHandlerFactory, SuggesterTest, TestBM25SimilarityFactory, DirectUpdateHandlerTest, DistribCursorPagingTest, FieldMutatingUpdateProcessorTest, RecoveryZkTest, TestNoOpRegenerator, TestSolrDeletionPolicy1, BasicDistributedZkTest, TestBinaryResponseWriter, TestRandomMergePolicy, ExternalFileFieldSortTest, OpenCloseCoreStressTest, PolyFieldTest, BasicDistributedZk2Test, TestFreeTextSuggestions, JsonLoaderTest, TestAddFieldRealTimeGet, UniqFieldsUpdateProcessorFactoryTest, TestAnalyzeInfixSuggestions, TestStressRecovery, TestBinaryField, IndexSchemaRuntimeFieldTest, TestSolrQueryParser, HdfsBasicDistributedZkTest, ZkNodePropsTest, BadComponentTest, UUIDFieldTest, AssignTest, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest]
>   [junit4] Completed in 443.02s, 1 test, 1 failure <<< FAILURES!
> 
> [...truncated 539 lines...]
> BUILD FAILED
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:459: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:439: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/build.xml:39: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/extra-targets.xml:37: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/build.xml:189: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/solr/common-build.xml:491: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:1308: The following error occurred while executing this line:
> /Users/jenkins/workspace/Lucene-Solr-4.x-MacOSX/lucene/common-build.xml:941: There were test failures: 354 suites, 1540 tests, 1 failure, 40 ignored (6 assumptions)
> 
> Total time: 85 minutes 49 seconds
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseSerialGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org