You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/27 12:44:31 UTC

[JENKINS] Lucene-Solr-trunk-MacOSX (64bit/jdk1.7.0) - Build # 771 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-MacOSX/771/
Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseParallelGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:49278/collection1, http://127.0.0.1:49275/collection1, http://127.0.0.1:49311/collection1, http://127.0.0.1:49272/collection1, http://127.0.0.1:49306/collection1, http://127.0.0.1:49299/collection1, http://127.0.0.1:49326/collection1, http://127.0.0.1:49292/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:49278/collection1, http://127.0.0.1:49275/collection1, http://127.0.0.1:49311/collection1, http://127.0.0.1:49272/collection1, http://127.0.0.1:49306/collection1, http://127.0.0.1:49299/collection1, http://127.0.0.1:49326/collection1, http://127.0.0.1:49292/collection1]
	at __randomizedtesting.SeedInfo.seed([8DDCD7B55E3817D8:C3A59AD296777E4]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:333)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:318)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1138)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:137)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:49272/collection1 returned non ok status:500, message:Server Error
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:298)
	... 46 more




Build Log:
[...truncated 9484 lines...]
   [junit4] Suite: org.apache.solr.cloud.ChaosMonkeySafeLeaderTest
   [junit4]   2> 186273 T426 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 186286 T426 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./solrtest-ChaosMonkeySafeLeaderTest-1377599469200
   [junit4]   2> 186288 T426 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 186288 T427 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 186389 T426 oasc.ZkTestServer.run start zk server on port:49265
   [junit4]   2> 186392 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186405 T433 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1eb605e7 name:ZooKeeperConnection Watcher:127.0.0.1:49265 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186405 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186406 T426 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 186416 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 186421 T435 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16cebd92 name:ZooKeeperConnection Watcher:127.0.0.1:49265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 186421 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 186422 T426 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 186430 T426 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 186437 T426 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 186443 T426 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 186451 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 186451 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 186464 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 186465 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 186474 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 186476 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 186485 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 186486 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 186497 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 186498 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 186510 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 186511 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 186524 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 186526 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 186538 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 186539 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 186554 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 186557 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 186572 T426 oasc.AbstractZkTestCase.putConfig put /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 186574 T426 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 187003 T426 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 187019 T426 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49268
   [junit4]   2> 187019 T426 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 187020 T426 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 187020 T426 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512
   [junit4]   2> 187020 T426 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/'
   [junit4]   2> 187053 T426 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/solr.xml
   [junit4]   2> 187173 T426 oasc.CoreContainer.<init> New CoreContainer 1130823356
   [junit4]   2> 187174 T426 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/]
   [junit4]   2> 187176 T426 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 187177 T426 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 187178 T426 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 187178 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 187179 T426 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 187180 T426 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 187180 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 187181 T426 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 187182 T426 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 187182 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 187205 T426 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49265/solr
   [junit4]   2> 187206 T426 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 187207 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 187213 T446 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34ac9c51 name:ZooKeeperConnection Watcher:127.0.0.1:49265 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 187213 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 187218 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 187242 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 187250 T448 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@70956cca name:ZooKeeperConnection Watcher:127.0.0.1:49265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 187251 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 187257 T426 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 187280 T426 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 187329 T426 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 187342 T426 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49268_
   [junit4]   2> 187344 T426 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49268_
   [junit4]   2> 187356 T426 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 187371 T426 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 187377 T426 oasc.Overseer.start Overseer (id=90282358814932995-127.0.0.1:49268_-n_0000000000) starting
   [junit4]   2> 187387 T426 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 187401 T450 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 187404 T426 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 187413 T426 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 187420 T426 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 187433 T449 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 187444 T451 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 187444 T451 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 187447 T451 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 188953 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 188957 T449 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49268_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49268"}
   [junit4]   2> 188958 T449 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 188959 T449 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 188983 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 189456 T451 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 189457 T451 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/collection1
   [junit4]   2> 189458 T451 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 189462 T451 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 189463 T451 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 189471 T451 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/collection1/'
   [junit4]   2> 189475 T451 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/collection1/lib/classes/' to classloader
   [junit4]   2> 189477 T451 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/collection1/lib/README' to classloader
   [junit4]   2> 189553 T451 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 189627 T451 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 189632 T451 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 189644 T451 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 190668 T451 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 190683 T451 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 190688 T451 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 190725 T451 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 190733 T451 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 190740 T451 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 190743 T451 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 190744 T451 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 190745 T451 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 190748 T451 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 190748 T451 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 190749 T451 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 190784 T451 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-controljetty-1377599469512/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/control/data/
   [junit4]   2> 190785 T451 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2e13cd9c
   [junit4]   2> 190786 T451 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/control/data
   [junit4]   2> 190787 T451 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/control/data/index/
   [junit4]   2> 190788 T451 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 190789 T451 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/control/data/index
   [junit4]   2> 190790 T451 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=37.603515625, floorSegmentMB=1.1826171875, forceMergeDeletesPctAllowed=16.29913977530185, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18176826910976562
   [junit4]   2> 190795 T451 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 190795 T451 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 190801 T451 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 190803 T451 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 190804 T451 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 190804 T451 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 190805 T451 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 190805 T451 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 190806 T451 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 190807 T451 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 190807 T451 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 190808 T451 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 190809 T451 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 190809 T451 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 190810 T451 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 190811 T451 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 190811 T451 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 190824 T451 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 190829 T451 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 190830 T451 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 190831 T451 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=93.9140625, floorSegmentMB=0.810546875, forceMergeDeletesPctAllowed=16.91377151791715, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 190833 T451 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/control/data/index,segFN=segments_1,generation=1}
   [junit4]   2> 190833 T451 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 190834 T451 oass.SolrIndexSearcher.<init> Opening Searcher@160e10f5 main
   [junit4]   2> 190844 T452 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@160e10f5 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 190846 T451 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 190846 T451 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49268 collection:control_collection shard:shard1
   [junit4]   2> 190847 T451 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 190864 T451 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 190882 T451 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 190889 T451 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 190890 T451 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 190890 T451 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49268/collection1/
   [junit4]   2> 190891 T451 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 190891 T451 oasc.SyncStrategy.syncToMe http://127.0.0.1:49268/collection1/ has no replicas
   [junit4]   2> 190892 T451 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49268/collection1/ shard1
   [junit4]   2> 190892 T451 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 192073 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 192112 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 192146 T451 oasc.ZkController.register We are http://127.0.0.1:49268/collection1/ and leader is http://127.0.0.1:49268/collection1/
   [junit4]   2> 192147 T451 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49268
   [junit4]   2> 192147 T451 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 192147 T451 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 192148 T451 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 192151 T451 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 192153 T426 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 192154 T426 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 192154 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 192169 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 192172 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192177 T455 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@ed19861 name:ZooKeeperConnection Watcher:127.0.0.1:49265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192178 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192182 T426 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 192189 T426 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 192569 T426 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 192576 T426 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49272
   [junit4]   2> 192579 T426 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 192580 T426 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 192580 T426 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103
   [junit4]   2> 192581 T426 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/'
   [junit4]   2> 192619 T426 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/solr.xml
   [junit4]   2> 192714 T426 oasc.CoreContainer.<init> New CoreContainer 968256266
   [junit4]   2> 192715 T426 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/]
   [junit4]   2> 192717 T426 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 192717 T426 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 192718 T426 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 192718 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 192719 T426 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 192719 T426 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 192720 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 192720 T426 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 192721 T426 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 192721 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 192742 T426 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49265/solr
   [junit4]   2> 192743 T426 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 192745 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192749 T466 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b4d147d name:ZooKeeperConnection Watcher:127.0.0.1:49265 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192750 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192754 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 192772 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192776 T468 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@262ad8e1 name:ZooKeeperConnection Watcher:127.0.0.1:49265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192777 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192788 T426 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 193636 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 193641 T449 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49268_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49268"}
   [junit4]   2> 193664 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193664 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193665 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193800 T426 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49272_
   [junit4]   2> 193808 T426 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49272_
   [junit4]   2> 193826 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 193832 T455 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 193832 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 193834 T468 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 193834 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 193844 T448 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 193860 T469 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 193860 T469 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 193863 T469 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 195191 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 195193 T449 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49272_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49272"}
   [junit4]   2> 195193 T449 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2, shard3]
   [junit4]   2> 195193 T449 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 195203 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 195204 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 195204 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 195869 T469 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 195869 T469 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/collection1
   [junit4]   2> 195870 T469 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 195873 T469 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 195873 T469 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 195879 T469 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/collection1/'
   [junit4]   2> 195882 T469 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/collection1/lib/classes/' to classloader
   [junit4]   2> 195883 T469 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/collection1/lib/README' to classloader
   [junit4]   2> 195966 T469 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 196047 T469 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 196051 T469 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 196065 T469 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 196765 T469 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 196779 T469 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 196784 T469 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 196822 T469 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 196832 T469 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 196841 T469 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 196844 T469 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 196844 T469 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 196845 T469 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 196848 T469 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 196848 T469 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 196849 T469 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 196849 T469 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty1-1377599475103/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty1/
   [junit4]   2> 196849 T469 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2e13cd9c
   [junit4]   2> 196851 T469 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty1
   [junit4]   2> 196852 T469 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty1/index/
   [junit4]   2> 196853 T469 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 196853 T469 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty1/index
   [junit4]   2> 196854 T469 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=37.603515625, floorSegmentMB=1.1826171875, forceMergeDeletesPctAllowed=16.29913977530185, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18176826910976562
   [junit4]   2> 196858 T469 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 196859 T469 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 196867 T469 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 196868 T469 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 196869 T469 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 196869 T469 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 196869 T469 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 196870 T469 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 196871 T469 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 196871 T469 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 196872 T469 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 196873 T469 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 196875 T469 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 196876 T469 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 196877 T469 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 196878 T469 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 196879 T469 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 196892 T469 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 196897 T469 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 196897 T469 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 196899 T469 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=93.9140625, floorSegmentMB=0.810546875, forceMergeDeletesPctAllowed=16.91377151791715, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 196901 T469 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty1/index,segFN=segments_1,generation=1}
   [junit4]   2> 196901 T469 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 196902 T469 oass.SolrIndexSearcher.<init> Opening Searcher@3f900a24 main
   [junit4]   2> 196912 T470 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3f900a24 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 196917 T469 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 196917 T469 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49272 collection:collection1 shard:shard1
   [junit4]   2> 196918 T469 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 196934 T469 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 196951 T469 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 196958 T469 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 196958 T469 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 196959 T469 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49272/collection1/
   [junit4]   2> 196959 T469 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 196959 T469 oasc.SyncStrategy.syncToMe http://127.0.0.1:49272/collection1/ has no replicas
   [junit4]   2> 196960 T469 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49272/collection1/ shard1
   [junit4]   2> 196960 T469 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 198230 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 198275 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198276 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198275 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198325 T469 oasc.ZkController.register We are http://127.0.0.1:49272/collection1/ and leader is http://127.0.0.1:49272/collection1/
   [junit4]   2> 198326 T469 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49272
   [junit4]   2> 198327 T469 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 198328 T469 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 198329 T469 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 198337 T469 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 198344 T426 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 198346 T426 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 198348 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 198813 T426 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 198819 T426 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49275
   [junit4]   2> 198821 T426 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 198822 T426 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 198822 T426 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286
   [junit4]   2> 198823 T426 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/'
   [junit4]   2> 198865 T426 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/solr.xml
   [junit4]   2> 198983 T426 oasc.CoreContainer.<init> New CoreContainer 448095337
   [junit4]   2> 198984 T426 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/]
   [junit4]   2> 198986 T426 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 198987 T426 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 198987 T426 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 198988 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 198988 T426 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 198989 T426 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 198989 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 198990 T426 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 198990 T426 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 198991 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 199013 T426 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49265/solr
   [junit4]   2> 199014 T426 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 199016 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 199021 T482 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e7607b6 name:ZooKeeperConnection Watcher:127.0.0.1:49265 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 199021 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 199029 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 199048 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 199053 T484 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@521c2cb9 name:ZooKeeperConnection Watcher:127.0.0.1:49265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 199054 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 199074 T426 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 199818 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 199823 T449 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49272_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49272"}
   [junit4]   2> 199848 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199849 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199848 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199849 T484 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 200087 T426 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49275_
   [junit4]   2> 200090 T426 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49275_
   [junit4]   2> 200095 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 200095 T484 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 200097 T468 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 200097 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 200098 T455 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 200098 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 200103 T448 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 200104 T484 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 200115 T485 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 200115 T485 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 200118 T485 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 201388 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 201393 T449 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49275_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49275"}
   [junit4]   2> 201394 T449 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 201395 T449 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 201423 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 201425 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 201427 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 201428 T484 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 202126 T485 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 202127 T485 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/collection1
   [junit4]   2> 202127 T485 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 202130 T485 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 202131 T485 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 202137 T485 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/collection1/'
   [junit4]   2> 202140 T485 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/collection1/lib/classes/' to classloader
   [junit4]   2> 202141 T485 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/collection1/lib/README' to classloader
   [junit4]   2> 202243 T485 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 202311 T485 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 202314 T485 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 202336 T485 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 203064 T485 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 203077 T485 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 203081 T485 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 203122 T485 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 203130 T485 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 203137 T485 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 203140 T485 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 203140 T485 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 203140 T485 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 203144 T485 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 203144 T485 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 203144 T485 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 203145 T485 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty2-1377599481286/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty2/
   [junit4]   2> 203145 T485 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2e13cd9c
   [junit4]   2> 203147 T485 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty2
   [junit4]   2> 203148 T485 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty2/index/
   [junit4]   2> 203149 T485 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 203149 T485 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty2/index
   [junit4]   2> 203150 T485 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=37.603515625, floorSegmentMB=1.1826171875, forceMergeDeletesPctAllowed=16.29913977530185, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18176826910976562
   [junit4]   2> 203155 T485 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 203155 T485 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 203160 T485 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 203161 T485 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 203161 T485 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 203161 T485 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 203162 T485 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 203162 T485 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 203163 T485 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 203163 T485 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 203164 T485 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 203164 T485 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 203166 T485 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 203166 T485 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 203166 T485 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 203167 T485 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 203168 T485 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 203178 T485 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 203182 T485 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 203182 T485 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 203183 T485 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=29, maxMergedSegmentMB=93.9140625, floorSegmentMB=0.810546875, forceMergeDeletesPctAllowed=16.91377151791715, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 203185 T485 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty2/index,segFN=segments_1,generation=1}
   [junit4]   2> 203186 T485 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 203186 T485 oass.SolrIndexSearcher.<init> Opening Searcher@1d1b8446 main
   [junit4]   2> 203194 T486 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1d1b8446 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 203197 T485 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 203197 T485 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49275 collection:collection1 shard:shard2
   [junit4]   2> 203198 T485 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 203212 T485 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 203228 T485 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 203235 T485 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 203235 T485 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 203236 T485 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49275/collection1/
   [junit4]   2> 203236 T485 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 203236 T485 oasc.SyncStrategy.syncToMe http://127.0.0.1:49275/collection1/ has no replicas
   [junit4]   2> 203237 T485 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49275/collection1/ shard2
   [junit4]   2> 203237 T485 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 204466 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 204508 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 204509 T484 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 204509 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 204509 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 204553 T485 oasc.ZkController.register We are http://127.0.0.1:49275/collection1/ and leader is http://127.0.0.1:49275/collection1/
   [junit4]   2> 204553 T485 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49275
   [junit4]   2> 204554 T485 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 204555 T485 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 204556 T485 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 204564 T485 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 204570 T426 oass.SolrDispatchFilter.init user.dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0
   [junit4]   2> 204571 T426 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 204572 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 204974 T426 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 204979 T426 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49278
   [junit4]   2> 204981 T426 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 204981 T426 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 204982 T426 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518
   [junit4]   2> 204982 T426 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/'
   [junit4]   2> 205020 T426 oasc.ConfigSolr.fromFile Loading container configuration from /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/solr.xml
   [junit4]   2> 205114 T426 oasc.CoreContainer.<init> New CoreContainer 1439062241
   [junit4]   2> 205115 T426 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/]
   [junit4]   2> 205117 T426 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 205118 T426 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 205118 T426 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 205119 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 205119 T426 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 205120 T426 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 205120 T426 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 205121 T426 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 205121 T426 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 205122 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 205141 T426 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49265/solr
   [junit4]   2> 205142 T426 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 205144 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 205149 T498 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@227f5ac9 name:ZooKeeperConnection Watcher:127.0.0.1:49265 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 205150 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 205156 T426 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 205186 T426 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 205190 T500 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@23b754a2 name:ZooKeeperConnection Watcher:127.0.0.1:49265/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 205191 T426 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 205206 T426 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 206035 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 206038 T449 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49275_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49275"}
   [junit4]   2> 206048 T500 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 206049 T484 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 206048 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 206049 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 206049 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 206218 T426 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49278_
   [junit4]   2> 206220 T426 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49278_
   [junit4]   2> 206225 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 206227 T484 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 206228 T500 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 206229 T500 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 206228 T468 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 206229 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 206230 T455 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 206230 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 206234 T448 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 206237 T484 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 206250 T501 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 206250 T501 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 206253 T501 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 207567 T449 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 207569 T449 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=3 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49278_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49278"}
   [junit4]   2> 207569 T449 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=3
   [junit4]   2> 207570 T449 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 207580 T500 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 207581 T468 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 207580 T448 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 207581 T484 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 207581 T455 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 208257 T501 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 208258 T501 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/collection1
   [junit4]   2> 208259 T501 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 208263 T501 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 208264 T501 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 208270 T501 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/collection1/'
   [junit4]   2> 208274 T501 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/collection1/lib/classes/' to classloader
   [junit4]   2> 208277 T501 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/collection1/lib/README' to classloader
   [junit4]   2> 208354 T501 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 208427 T501 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 208430 T501 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 208451 T501 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 209197 T501 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 209212 T501 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 209216 T501 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 209257 T501 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 209263 T501 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 209268 T501 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 209271 T501 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 209271 T501 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 209271 T501 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 209274 T501 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 209274 T501 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 209274 T501 oasc.SolrCore.initDirectoryFactory solr.StandardDirectoryFactory
   [junit4]   2> 209275 T501 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-jetty3-1377599487518/collection1/, dataDir=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty3/
   [junit4]   2> 209275 T501 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2e13cd9c
   [junit4]   2> 209277 T501 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty3
   [junit4]   2> 209278 T501 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty3/index/
   [junit4]   2> 209278 T501 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 209279 T501 oasc.CachingDirectoryFactory.get return new directory for /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty3/index
   [junit4]   2> 209280 T501 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=12, maxMergeAtOnceExplicit=26, maxMergedSegmentMB=37.603515625, floorSegmentMB=1.1826171875, forceMergeDeletesPctAllowed=16.29913977530185, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.18176826910976562
   [junit4]   2> 209283 T501 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/J0/org.apache.solr.cloud.ChaosMonkeySafeLeaderTest-1377599469200/jetty3/index,segFN=segments_1,generation=1}
   [junit4]   2> 209283 T501 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 209289 T501 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 209290 T501 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 209290 T501 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 209290 T501 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 209292 T501 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 209292 T501 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 209293 T501 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 209294 T501 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 209294 T501 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 209295 T501 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 209296 T501 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 209296 T501 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 209297 T501 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 209297 T501 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 209298 T501 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 209311 T501 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 209315 T501 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 209316 T501 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 209317 T501 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, 

[...truncated too long message...]

queryparser-5.0-SNAPSHOT.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/build/join/lucene-join-5.0-SNAPSHOT.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/commons-cli-1.2.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/commons-codec-1.7.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/commons-configuration-1.6.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/commons-fileupload-1.2.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/commons-lang-2.6.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/concurrentlinkedhashmap-lru-1.2.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/dom4j-1.6.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/guava-14.0.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/hadoop-annotations-2.0.5-alpha.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/hadoop-auth-2.0.5-alpha.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/hadoop-common-2.0.5-alpha.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/hadoop-hdfs-2.0.5-alpha.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/joda-time-2.2.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/org.restlet-2.1.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/org.restlet.ext.servlet-2.1.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/protobuf-java-2.4.0a.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/lib/spatial4j-0.3.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/commons-io-2.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/httpclient-4.2.3.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/httpcore-4.2.2.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/httpmime-4.2.3.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/jcl-over-slf4j-1.6.6.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/jul-to-slf4j-1.6.6.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/log4j-1.2.16.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/noggit-0.5.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/slf4j-api-1.6.6.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/slf4j-log4j12-1.6.6.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/wstx-asl-3.2.7.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/solrj/lib/zookeeper-3.4.5.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-continuation-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-deploy-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-http-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-io-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-jmx-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-security-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-server-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-servlet-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-util-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-webapp-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/jetty-xml-8.1.10.v20130312.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/lib/servlet-api-3.0.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/example-DIH/solr/db/lib/derby-10.9.1.0.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/example/example-DIH/solr/db/lib/hsqldb-1.8.0.10.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/build/core/classes/java:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/test-framework/lib/junit-4.10.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/test-framework/lib/randomizedtesting-runner-2.0.10.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/cglib-nodep-2.2.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/commons-collections-3.2.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/dom4j-1.6.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/easymock-3.0.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/hadoop-common-2.0.5-alpha-tests.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/hadoop-hdfs-2.0.5-alpha-tests.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/javax.servlet-api-3.0.1.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/jersey-core-1.16.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/jetty-6.1.26.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/jetty-util-6.1.26.jar:/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/core/test-lib/objenesis-1.2.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-launcher.jar:/Users/jenkins/.ant/lib/ivy-2.3.0.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-antlr.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bcel.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-bsf.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-log4j.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-oro.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-regexp.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-resolver.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-apache-xalan2.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-logging.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-commons-net.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jai.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-javamail.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jdepend.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jmf.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-jsch.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-junit4.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-netrexx.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-swing.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant-testutil.jar:/Users/jenkins/tools/hudson.tasks.Ant_AntInstallation/ANT_1.8.2/lib/ant.jar:/Library/Java/JavaVirtualMachines/jdk1.7.0_25.jdk/Contents/Home/lib/tools.jar:/Users/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.0.10.jar -ea:org.apache.lucene... -ea:org.apache.solr... com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -flush -eventsfile /Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/temp/junit4-J0-20130827_102758_671.events @/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build/solr-core/test/temp/junit4-J0-20130827_102758_671.suites
   [junit4] ERROR: JVM J0 ended with an exception: Forked process returned with error code: 134. Very likely a JVM crash.  Process output piped in logs above.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1254)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:67)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:822)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:819)
   [junit4] 	at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:166)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4] 	at java.lang.Thread.run(Thread.java:724)

BUILD FAILED
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:389: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:369: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/build.xml:39: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/build.xml:181: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/solr/common-build.xml:450: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:1230: The following error occurred while executing this line:
/Users/jenkins/workspace/Lucene-Solr-trunk-MacOSX/lucene/common-build.xml:873: At least one slave process threw an exception, first: Forked process returned with error code: 134. Very likely a JVM crash.  Process output piped in logs above.

Total time: 51 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.7.0 -XX:-UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure