You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/04 20:31:02 UTC

[JENKINS] Lucene-Solr-4.x-Linux (32bit/ibm-j9-jdk6) - Build # 6794 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/6794/
Java: 32bit/ibm-j9-jdk6 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
Server at http://127.0.0.1:41273/bz_s/vs/onenodecollectioncore returned non ok status:404, message:Can not find: /bz_s/vs/onenodecollectioncore/update

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:41273/bz_s/vs/onenodecollectioncore returned non ok status:404, message:Can not find: /bz_s/vs/onenodecollectioncore/update
	at __randomizedtesting.SeedInfo.seed([FCE931C06B23EAC8:7D0FBFD81C7C8AF4]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
	at java.lang.reflect.Method.invoke(Method.java:611)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:738)




Build Log:
[...truncated 9206 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZk2Test
   [junit4]   2> 950448 T2399 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /bz_s/vs
   [junit4]   2> 950454 T2399 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-BasicDistributedZk2Test-1375640258520
   [junit4]   2> 950455 T2399 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 950457 T2400 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 950557 T2399 oasc.ZkTestServer.run start zk server on port:58586
   [junit4]   2> 950558 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 950692 T2406 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b760b76 name:ZooKeeperConnection Watcher:127.0.0.1:58586 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 950692 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 950693 T2399 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 950701 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 950703 T2408 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@680c680c name:ZooKeeperConnection Watcher:127.0.0.1:58586/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 950704 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 950704 T2399 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 950712 T2399 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 950720 T2399 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 950727 T2399 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 950735 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 950736 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 950747 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 950748 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 950756 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 950757 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 950765 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 950767 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 950774 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 950775 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 950784 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 950785 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 950793 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 950794 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 950802 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 950803 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 950811 T2399 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 950812 T2399 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 951026 T2399 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 951028 T2399 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:38975
   [junit4]   2> 951029 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 951030 T2399 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 951031 T2399 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887
   [junit4]   2> 951031 T2399 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/'
   [junit4]   2> 951043 T2399 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/solr.xml
   [junit4]   2> 951080 T2399 oasc.CoreContainer.<init> New CoreContainer 1856925358
   [junit4]   2> 951081 T2399 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/]
   [junit4]   2> 951083 T2399 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 951083 T2399 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 951084 T2399 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 951085 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 951086 T2399 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 951086 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 951087 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 951088 T2399 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 951089 T2399 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 951090 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 951097 T2399 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58586/solr
   [junit4]   2> 951098 T2399 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 951099 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 951102 T2419 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f720f72 name:ZooKeeperConnection Watcher:127.0.0.1:58586 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 951103 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 951107 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 951114 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 951116 T2421 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b744b74 name:ZooKeeperConnection Watcher:127.0.0.1:58586/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 951117 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 951121 T2399 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 951134 T2399 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 951149 T2399 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 951156 T2399 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38975_bz_s%2Fvs
   [junit4]   2> 951159 T2399 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38975_bz_s%2Fvs
   [junit4]   2> 951172 T2399 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 951191 T2399 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 951200 T2399 oasc.Overseer.start Overseer (id=90153959982759939-127.0.0.1:38975_bz_s%2Fvs-n_0000000000) starting
   [junit4]   2> 951220 T2399 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 951234 T2423 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 951236 T2399 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 951246 T2399 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 951253 T2399 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 951266 T2422 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 951271 T2424 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 951271 T2424 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 951275 T2424 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 952782 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 952786 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38975/bz_s/vs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38975_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 952787 T2422 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 952788 T2422 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 952799 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 953276 T2424 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 953277 T2424 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/collection1
   [junit4]   2> 953277 T2424 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 953280 T2424 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 953280 T2424 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 953284 T2424 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/collection1/'
   [junit4]   2> 953286 T2424 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/collection1/lib/README' to classloader
   [junit4]   2> 953287 T2424 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/collection1/lib/classes/' to classloader
   [junit4]   2> 953332 T2424 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 953383 T2424 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 953387 T2424 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 953396 T2424 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 953797 T2424 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 953798 T2424 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 953799 T2424 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 953804 T2424 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 953807 T2424 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 953856 T2424 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 953865 T2424 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 953881 T2424 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 953886 T2424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 953887 T2424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 953888 T2424 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 953893 T2424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 953894 T2424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 953895 T2424 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 953895 T2424 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1375640258887/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/control/data/
   [junit4]   2> 953896 T2424 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@302b302b
   [junit4]   2> 953897 T2424 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/control/data
   [junit4]   2> 953898 T2424 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/control/data/index/
   [junit4]   2> 953898 T2424 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 953899 T2424 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/control/data/index
   [junit4]   2> 953902 T2424 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@1ab61ab6 lockFactory=org.apache.lucene.store.NativeFSLockFactory@13ef13ef),segFN=segments_1,generation=1}
   [junit4]   2> 953903 T2424 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 953911 T2424 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 953912 T2424 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 953913 T2424 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 953914 T2424 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 953915 T2424 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 953916 T2424 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 953917 T2424 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 953918 T2424 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 953919 T2424 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 953920 T2424 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 953923 T2424 oass.SolrIndexSearcher.<init> Opening Searcher@47c847c8 main
   [junit4]   2> 953924 T2424 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 953924 T2424 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 953931 T2425 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@47c847c8 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 953934 T2424 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 953935 T2424 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38975/bz_s/vs collection:control_collection shard:shard1
   [junit4]   2> 953936 T2424 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 953944 T2424 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 953967 T2424 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 953975 T2424 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 953976 T2424 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 953976 T2424 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38975/bz_s/vs/collection1/
   [junit4]   2> 953977 T2424 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 953978 T2424 oasc.SyncStrategy.syncToMe http://127.0.0.1:38975/bz_s/vs/collection1/ has no replicas
   [junit4]   2> 953978 T2424 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:38975/bz_s/vs/collection1/ shard1
   [junit4]   2> 953979 T2424 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 954318 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 954346 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 954401 T2424 oasc.ZkController.register We are http://127.0.0.1:38975/bz_s/vs/collection1/ and leader is http://127.0.0.1:38975/bz_s/vs/collection1/
   [junit4]   2> 954402 T2424 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:38975/bz_s/vs
   [junit4]   2> 954402 T2424 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 954403 T2424 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 954403 T2424 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 954407 T2424 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 954410 T2399 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 954411 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 954412 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 954418 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 954420 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 954423 T2428 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39f339f3 name:ZooKeeperConnection Watcher:127.0.0.1:58586/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 954425 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 954434 T2399 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 954443 T2399 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 954648 T2399 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 954650 T2399 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:36173
   [junit4]   2> 954651 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 954652 T2399 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 954652 T2399 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509
   [junit4]   2> 954653 T2399 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/'
   [junit4]   2> 954665 T2399 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/solr.xml
   [junit4]   2> 954701 T2399 oasc.CoreContainer.<init> New CoreContainer 233180646
   [junit4]   2> 954701 T2399 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/]
   [junit4]   2> 954703 T2399 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 954704 T2399 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 954704 T2399 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 954705 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 954706 T2399 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 954707 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 954707 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 954708 T2399 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 954709 T2399 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 954710 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 954717 T2399 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58586/solr
   [junit4]   2> 954718 T2399 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 954719 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 954721 T2439 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34c334c3 name:ZooKeeperConnection Watcher:127.0.0.1:58586 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 954722 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 954726 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 954733 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 954735 T2441 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4dfa4dfa name:ZooKeeperConnection Watcher:127.0.0.1:58586/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 954736 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 954748 T2399 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 955760 T2399 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36173_bz_s%2Fvs
   [junit4]   2> 955762 T2399 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36173_bz_s%2Fvs
   [junit4]   2> 955768 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 955780 T2428 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 955780 T2441 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 955786 T2421 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 955798 T2442 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 955798 T2442 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 955801 T2442 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 955877 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 955885 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38975/bz_s/vs",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:38975_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 955898 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36173/bz_s/vs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:36173_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 955899 T2422 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 955900 T2422 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 955911 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 955912 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 955911 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 956802 T2442 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 956803 T2442 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/collection1
   [junit4]   2> 956803 T2442 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 956805 T2442 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 956806 T2442 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 956810 T2442 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/collection1/'
   [junit4]   2> 956812 T2442 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/collection1/lib/README' to classloader
   [junit4]   2> 956813 T2442 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/collection1/lib/classes/' to classloader
   [junit4]   2> 956852 T2442 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 956887 T2442 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 956893 T2442 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 956900 T2442 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 957291 T2442 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 957292 T2442 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 957293 T2442 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 957298 T2442 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 957301 T2442 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 957346 T2442 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 957352 T2442 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 957359 T2442 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 957363 T2442 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 957364 T2442 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 957364 T2442 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 957368 T2442 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 957369 T2442 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 957369 T2442 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 957370 T2442 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1375640262509/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty1/
   [junit4]   2> 957370 T2442 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@302b302b
   [junit4]   2> 957372 T2442 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty1
   [junit4]   2> 957372 T2442 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty1/index/
   [junit4]   2> 957373 T2442 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 957374 T2442 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty1/index
   [junit4]   2> 957376 T2442 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@521e521e lockFactory=org.apache.lucene.store.NativeFSLockFactory@2fa32fa3),segFN=segments_1,generation=1}
   [junit4]   2> 957376 T2442 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 957384 T2442 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 957385 T2442 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 957385 T2442 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 957386 T2442 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 957387 T2442 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 957388 T2442 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 957388 T2442 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 957389 T2442 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 957390 T2442 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 957392 T2442 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 957394 T2442 oass.SolrIndexSearcher.<init> Opening Searcher@4c194c19 main
   [junit4]   2> 957395 T2442 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 957396 T2442 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 957403 T2443 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4c194c19 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 957408 T2442 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 957408 T2442 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36173/bz_s/vs collection:collection1 shard:shard1
   [junit4]   2> 957409 T2442 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 957417 T2442 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 957441 T2442 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 957448 T2442 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 957448 T2442 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 957449 T2442 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36173/bz_s/vs/collection1/
   [junit4]   2> 957450 T2442 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 957450 T2442 oasc.SyncStrategy.syncToMe http://127.0.0.1:36173/bz_s/vs/collection1/ has no replicas
   [junit4]   2> 957451 T2442 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36173/bz_s/vs/collection1/ shard1
   [junit4]   2> 957451 T2442 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 958944 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 959111 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 959112 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 959112 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 959153 T2442 oasc.ZkController.register We are http://127.0.0.1:36173/bz_s/vs/collection1/ and leader is http://127.0.0.1:36173/bz_s/vs/collection1/
   [junit4]   2> 959154 T2442 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36173/bz_s/vs
   [junit4]   2> 959154 T2442 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 959155 T2442 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 959155 T2442 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 959159 T2442 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 959162 T2399 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 959163 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 959164 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 959349 T2399 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 959351 T2399 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41273
   [junit4]   2> 959352 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 959353 T2399 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 959354 T2399 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234
   [junit4]   2> 959355 T2399 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/'
   [junit4]   2> 959370 T2399 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/solr.xml
   [junit4]   2> 959412 T2399 oasc.CoreContainer.<init> New CoreContainer 1802857333
   [junit4]   2> 959413 T2399 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/]
   [junit4]   2> 959414 T2399 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 959415 T2399 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 959416 T2399 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 959417 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 959418 T2399 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 959419 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 959420 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 959421 T2399 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 959422 T2399 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 959423 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 959433 T2399 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58586/solr
   [junit4]   2> 959434 T2399 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 959435 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 959447 T2455 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7320732 name:ZooKeeperConnection Watcher:127.0.0.1:58586 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 959448 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 959455 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 959463 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 959466 T2457 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64626462 name:ZooKeeperConnection Watcher:127.0.0.1:58586/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 959467 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 959486 T2399 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 960501 T2399 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41273_bz_s%2Fvs
   [junit4]   2> 960503 T2399 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41273_bz_s%2Fvs
   [junit4]   2> 960509 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 960509 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 960510 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 960515 T2457 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 960528 T2421 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 960529 T2428 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 960528 T2441 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 960539 T2458 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 960539 T2458 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 960541 T2458 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 960636 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 960640 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36173/bz_s/vs",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:36173_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 960690 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41273/bz_s/vs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:41273_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 960691 T2422 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 960692 T2422 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 960712 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 960712 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 960713 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 960712 T2457 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 961542 T2458 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 961543 T2458 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/collection1
   [junit4]   2> 961543 T2458 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 961546 T2458 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 961547 T2458 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 961552 T2458 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/collection1/'
   [junit4]   2> 961553 T2458 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/collection1/lib/README' to classloader
   [junit4]   2> 961554 T2458 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/collection1/lib/classes/' to classloader
   [junit4]   2> 961589 T2458 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 961625 T2458 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 961630 T2458 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 961637 T2458 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 962064 T2458 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 962065 T2458 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 962065 T2458 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 962071 T2458 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 962074 T2458 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 962122 T2458 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 962129 T2458 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 962136 T2458 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 962140 T2458 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 962141 T2458 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 962142 T2458 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 962146 T2458 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 962146 T2458 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 962147 T2458 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 962147 T2458 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1375640267234/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty2/
   [junit4]   2> 962148 T2458 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@302b302b
   [junit4]   2> 962149 T2458 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty2
   [junit4]   2> 962150 T2458 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty2/index/
   [junit4]   2> 962150 T2458 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 962151 T2458 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty2/index
   [junit4]   2> 962166 T2458 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@28682868 lockFactory=org.apache.lucene.store.NativeFSLockFactory@5ac85ac8),segFN=segments_1,generation=1}
   [junit4]   2> 962167 T2458 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 962174 T2458 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 962175 T2458 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 962176 T2458 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 962177 T2458 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 962178 T2458 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 962178 T2458 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 962179 T2458 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 962180 T2458 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 962181 T2458 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 962182 T2458 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 962185 T2458 oass.SolrIndexSearcher.<init> Opening Searcher@72507250 main
   [junit4]   2> 962186 T2458 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 962186 T2458 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 962194 T2459 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@72507250 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 962198 T2458 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 962199 T2458 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41273/bz_s/vs collection:collection1 shard:shard2
   [junit4]   2> 962199 T2458 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 962211 T2458 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 962233 T2458 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 962244 T2458 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 962245 T2458 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 962246 T2458 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41273/bz_s/vs/collection1/
   [junit4]   2> 962247 T2458 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 962247 T2458 oasc.SyncStrategy.syncToMe http://127.0.0.1:41273/bz_s/vs/collection1/ has no replicas
   [junit4]   2> 962248 T2458 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41273/bz_s/vs/collection1/ shard2
   [junit4]   2> 962249 T2458 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 963751 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 963784 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 963785 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 963784 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 963787 T2457 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 963819 T2458 oasc.ZkController.register We are http://127.0.0.1:41273/bz_s/vs/collection1/ and leader is http://127.0.0.1:41273/bz_s/vs/collection1/
   [junit4]   2> 963820 T2458 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41273/bz_s/vs
   [junit4]   2> 963821 T2458 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 963821 T2458 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 963822 T2458 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 963828 T2458 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 963831 T2399 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 963832 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 963869 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 964045 T2399 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 964048 T2399 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:53451
   [junit4]   2> 964049 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 964050 T2399 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 964052 T2399 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940
   [junit4]   2> 964053 T2399 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/'
   [junit4]   2> 964084 T2399 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/solr.xml
   [junit4]   2> 964126 T2399 oasc.CoreContainer.<init> New CoreContainer 183765748
   [junit4]   2> 964127 T2399 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/]
   [junit4]   2> 964128 T2399 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 964129 T2399 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 964130 T2399 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 964131 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 964132 T2399 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 964133 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 964133 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 964134 T2399 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 964135 T2399 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 964136 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 964144 T2399 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58586/solr
   [junit4]   2> 964145 T2399 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 964147 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 964153 T2471 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f2d6f2d name:ZooKeeperConnection Watcher:127.0.0.1:58586 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 964154 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 964159 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 964165 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 964167 T2473 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d607d6 name:ZooKeeperConnection Watcher:127.0.0.1:58586/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 964168 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 964182 T2399 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 965193 T2399 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53451_bz_s%2Fvs
   [junit4]   2> 965196 T2399 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53451_bz_s%2Fvs
   [junit4]   2> 965201 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 965203 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 965204 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 965207 T2473 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 965208 T2457 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 965209 T2457 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 965216 T2428 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 965219 T2421 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 965224 T2441 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 965237 T2474 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 965237 T2474 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 965240 T2474 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 965346 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 965351 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41273/bz_s/vs",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:41273_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 965363 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53451/bz_s/vs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "node_name":"127.0.0.1:53451_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 965364 T2422 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 965365 T2422 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 965375 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 965376 T2473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 965376 T2457 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 965376 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 965381 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 966240 T2474 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 966241 T2474 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/collection1
   [junit4]   2> 966241 T2474 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 966243 T2474 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 966244 T2474 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 966248 T2474 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/collection1/'
   [junit4]   2> 966249 T2474 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/collection1/lib/README' to classloader
   [junit4]   2> 966250 T2474 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/collection1/lib/classes/' to classloader
   [junit4]   2> 966293 T2474 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 966335 T2474 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 966352 T2474 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 966359 T2474 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 966781 T2474 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 966781 T2474 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 966782 T2474 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 966790 T2474 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 966793 T2474 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 966836 T2474 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 966843 T2474 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 966850 T2474 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 966855 T2474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 966855 T2474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 966856 T2474 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 966860 T2474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 966861 T2474 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 966861 T2474 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 966862 T2474 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1375640271940/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty3/
   [junit4]   2> 966863 T2474 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@302b302b
   [junit4]   2> 966864 T2474 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty3
   [junit4]   2> 966864 T2474 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty3/index/
   [junit4]   2> 966865 T2474 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 966866 T2474 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1375640258520/jetty3/index
   [junit4]   2> 966868 T2474 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@45e045e0 lockFactory=org.apache.lucene.store.NativeFSLockFactory@29842984),segFN=segments_1,generation=1}
   [junit4]   2> 966869 T2474 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 966878 T2474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 966879 T2474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 966880 T2474 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 966881 T2474 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 966882 T2474 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 966883 T2474 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 966884 T2474 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 966885 T2474 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 966886 T2474 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 966887 T2474 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 966903 T2474 oass.SolrIndexSearcher.<init> Opening Searcher@70c670c6 main
   [junit4]   2> 966905 T2474 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 966906 T2474 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 966914 T2475 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@70c670c6 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 966919 T2474 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 966919 T2474 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:53451/bz_s/vs collection:collection1 shard:shard1
   [junit4]   2> 966920 T2474 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 966939 T2474 oasc.ZkController.register We are http://127.0.0.1:53451/bz_s/vs/collection1/ and leader is http://127.0.0.1:36173/bz_s/vs/collection1/
   [junit4]   2> 966940 T2474 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:53451/bz_s/vs
   [junit4]   2> 966941 T2474 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 966941 T2474 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C6532 name=collection1 org.apache.solr.core.SolrCore@1baf1baf url=http://127.0.0.1:53451/bz_s/vs/collection1 node=127.0.0.1:53451_bz_s%2Fvs C6532_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:53451/bz_s/vs, state=down, node_name=127.0.0.1:53451_bz_s%2Fvs, core=collection1}
   [junit4]   2> 966942 T2476 C6532 P53451 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 966942 T2476 C6532 P53451 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 966943 T2476 C6532 P53451 oasc.ZkController.publish publishing core=collection1 state=recovering
   [junit4]   2> 966943 T2476 C6532 P53451 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 966944 T2474 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 966947 T2476 C6532 P53451 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 966949 T2399 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 966950 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 966951 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 966959 T2437 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true
   [junit4]   2> 967171 T2399 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 967172 T2399 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:42489
   [junit4]   2> 967174 T2399 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 967174 T2399 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 967175 T2399 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1375640275022
   [junit4]   2> 967176 T2399 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1375640275022/'
   [junit4]   2> 967189 T2399 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1375640275022/solr.xml
   [junit4]   2> 967226 T2399 oasc.CoreContainer.<init> New CoreContainer 1882288177
   [junit4]   2> 967227 T2399 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1375640275022/]
   [junit4]   2> 967228 T2399 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 967229 T2399 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 967230 T2399 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 967230 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 967231 T2399 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 967232 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 967233 T2399 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 967233 T2399 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 967234 T2399 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 967235 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 967242 T2399 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58586/solr
   [junit4]   2> 967244 T2399 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 967245 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 967247 T2488 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27cc27cc name:ZooKeeperConnection Watcher:127.0.0.1:58586 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 967248 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 967253 T2399 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 967260 T2399 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 967262 T2490 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a104a1 name:ZooKeeperConnection Watcher:127.0.0.1:58586/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 967263 T2399 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 967296 T2399 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 968314 T2399 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42489_bz_s%2Fvs
   [junit4]   2> 968318 T2399 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42489_bz_s%2Fvs
   [junit4]   2> 968327 T2441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 968328 T2428 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 968328 T2421 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 968329 T2473 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 968332 T2457 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 968333 T2457 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 968345 T2490 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 968354 T2473 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 968358 T2428 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 968359 T2421 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 968364 T2441 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 968381 T2491 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 968382 T2491 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 968385 T2491 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 968421 T2422 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 968426 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53451/bz_s/vs",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "node_name":"127.0.0.1:53451_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 968444 T2422 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42489/bz_s/vs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node4",
   [junit4]   2> 	  "node_name":"127.0.0.1:42489_bz_s%2Fvs",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"coll

[...truncated too long message...]

ect/shard1/election/90153959982759942-core_node1-n_0000000000 (0)
   [junit4]   1>     /solr/collections/onenodecollection/leaders (1)
   [junit4]   1>      /solr/collections/onenodecollection/leaders/shard1 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "base_url":"http://127.0.0.1:36173/bz_s/vs",
   [junit4]   1>            "node_name":"127.0.0.1:36173_bz_s%2Fvs",
   [junit4]   1>            "core":"onenodecollectioncore"}
   [junit4]   1>   /solr/aliases.json (0)
   [junit4]   1>   /solr/live_nodes (5)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:53451_bz_s%2Fvs (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:36173_bz_s%2Fvs (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:38975_bz_s%2Fvs (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:42489_bz_s%2Fvs (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:41273_bz_s%2Fvs (0)
   [junit4]   1>   /solr/clusterstate.json (0)
   [junit4]   1>   DATA:
   [junit4]   1>       {
   [junit4]   1>         "control_collection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:38975/bz_s/vs",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:38975_bz_s%2Fvs",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "onenodecollection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:36173/bz_s/vs",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:36173_bz_s%2Fvs",
   [junit4]   1>                   "roles":"none",
   [junit4]   1>                   "core":"onenodecollectioncore",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "collection1":{
   [junit4]   1>           "shards":{
   [junit4]   1>             "shard1":{
   [junit4]   1>               "range":"80000000-ffffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:36173/bz_s/vs",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:36173_bz_s%2Fvs",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node3":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:53451/bz_s/vs",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:53451_bz_s%2Fvs",
   [junit4]   1>                   "core":"collection1"}}},
   [junit4]   1>             "shard2":{
   [junit4]   1>               "range":"0-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node2":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:41273/bz_s/vs",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:41273_bz_s%2Fvs",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node4":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:42489/bz_s/vs",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:42489_bz_s%2Fvs",
   [junit4]   1>                   "core":"collection1"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"}}
   [junit4]   1>  /zookeeper (1)
   [junit4]   1>  DATA:
   [junit4]   1>      
   [junit4]   1> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZk2Test -Dtests.method=testDistribSearch -Dtests.seed=FCE931C06B23EAC8 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_JO -Dtests.timezone=Mideast/Riyadh88 -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   34.2s J1 | BasicDistributedZk2Test.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:41273/bz_s/vs/onenodecollectioncore returned non ok status:404, message:Can not find: /bz_s/vs/onenodecollectioncore/update
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([FCE931C06B23EAC8:7D0FBFD81C7C8AF4]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:738)
   [junit4]   2> 984690 T2399 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 34261 T2398 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> 985522 T2490 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 985523 T2490 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene3x, sim=DefaultSimilarity, locale=ar_JO, timezone=Mideast/Riyadh88
   [junit4]   2> NOTE: Linux 3.2.0-51-generic x86/IBM Corporation 1.6.0 (32-bit)/cpus=8,threads=3,free=19805816,total=56882176
   [junit4]   2> NOTE: All tests run in this JVM: [TestArbitraryIndexDir, SampleTest, TestIndexSearcher, DocValuesMultiTest, ClusterStateTest, TestCodecSupport, CurrencyFieldOpenExchangeTest, DirectSolrSpellCheckerTest, DirectUpdateHandlerTest, SolrPluginUtilsTest, TestBM25SimilarityFactory, TestCopyFieldCollectionResource, OutputWriterTest, TestQuerySenderNoQuery, TestCSVResponseWriter, ChaosMonkeySafeLeaderTest, ZkNodePropsTest, StressHdfsTest, TestBinaryResponseWriter, CurrencyFieldXmlFileTest, DistanceFunctionTest, BlockCacheTest, SpellPossibilityIteratorTest, TestSolrXMLSerializer, IndexReaderFactoryTest, TestRangeQuery, TestQueryTypes, SolrIndexConfigTest, IndexBasedSpellCheckerTest, PingRequestHandlerTest, TestBinaryField, TestInfoStreamLogging, HdfsDirectoryTest, NoCacheHeaderTest, DebugComponentTest, TestJmxMonitoredMap, SolrCoreCheckLockOnStartupTest, UpdateParamsTest, TestDistributedSearch, BlockDirectoryTest, ConvertedLegacyTest, HighlighterConfigTest, SignatureUpdateProcessorFactoryTest, NotRequiredUniqueKeyTest, IndexSchemaRuntimeFieldTest, TestMultiCoreConfBootstrap, HdfsLockFactoryTest, TestUtils, LeaderElectionIntegrationTest, TestMergePolicyConfig, TestPartialUpdateDeduplication, TermVectorComponentDistributedTest, TestFieldResource, EchoParamsTest, TestDefaultSearchFieldResource, TestFiltering, SimplePostToolTest, NumericFieldsTest, BadCopyFieldTest, DOMUtilTest, TestZkChroot, HdfsUnloadDistributedZkTest, TestAnalyzedSuggestions, JSONWriterTest, TestCollationKeyRangeQueries, TestDistributedGrouping, FullSolrCloudDistribCmdsTest, TestCoreDiscovery, OpenCloseCoreStressTest, RequiredFieldsTest, FileUtilsTest, TestCloudManagedSchemaAddField, BasicDistributedZkTest, BasicDistributedZk2Test]
   [junit4] Completed on J1 in 35.29s, 1 test, 1 error <<< FAILURES!

[...truncated 525 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:395: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:388: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:181: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:449: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1233: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:876: There were test failures: 321 suites, 1351 tests, 1 error, 34 ignored (6 assumptions)

Total time: 53 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/ibm-j9-jdk6 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure