You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/16 18:04:22 UTC

[JENKINS] Lucene-Solr-4.x-Linux (32bit/ibm-j9-jdk6) - Build # 6955 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/6955/
Java: 32bit/ibm-j9-jdk6 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
Server at http://127.0.0.1:40942/fi_zy/onenodecollectioncore returned non ok status:404, message:Can not find: /fi_zy/onenodecollectioncore/update

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:40942/fi_zy/onenodecollectioncore returned non ok status:404, message:Can not find: /fi_zy/onenodecollectioncore/update
	at __randomizedtesting.SeedInfo.seed([3EE244D1B8580015:BF04CAC9CF076029]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
	at java.lang.reflect.Method.invoke(Method.java:611)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:738)




Build Log:
[...truncated 9373 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZk2Test
   [junit4]   2> 1037754 T2285 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /fi_zy/
   [junit4]   2> 1037757 T2285 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solrtest-BasicDistributedZk2Test-1376668277684
   [junit4]   2> 1037758 T2285 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1037759 T2286 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1037859 T2285 oasc.ZkTestServer.run start zk server on port:50098
   [junit4]   2> 1037862 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038318 T2292 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@29092909 name:ZooKeeperConnection Watcher:127.0.0.1:50098 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038318 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038319 T2285 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1038329 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038331 T2294 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@241a241a name:ZooKeeperConnection Watcher:127.0.0.1:50098/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038332 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038332 T2285 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1038340 T2285 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1038347 T2285 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1038354 T2285 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1038361 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1038362 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1038373 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1038374 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1038383 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1038384 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1038391 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1038392 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1038400 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1038401 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1038410 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1038411 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1038419 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1038420 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1038427 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1038428 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1038460 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1038461 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1038472 T2285 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1038473 T2285 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1038659 T2285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1038663 T2285 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:39725
   [junit4]   2> 1038664 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1038665 T2285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1038667 T2285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412
   [junit4]   2> 1038668 T2285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/'
   [junit4]   2> 1038685 T2285 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/solr.xml
   [junit4]   2> 1038727 T2285 oasc.CoreContainer.<init> New CoreContainer 1318342292
   [junit4]   2> 1038728 T2285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/]
   [junit4]   2> 1038729 T2285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1038730 T2285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1038731 T2285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1038732 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1038732 T2285 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1038733 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1038734 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1038735 T2285 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1038735 T2285 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1038736 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1038744 T2285 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50098/solr
   [junit4]   2> 1038745 T2285 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1038746 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038749 T2305 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7cef7cef name:ZooKeeperConnection Watcher:127.0.0.1:50098 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038749 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038755 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1038762 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038764 T2307 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2d592d59 name:ZooKeeperConnection Watcher:127.0.0.1:50098/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038764 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038768 T2285 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1038780 T2285 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1038791 T2285 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1038797 T2285 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39725_fi_zy
   [junit4]   2> 1038799 T2285 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39725_fi_zy
   [junit4]   2> 1038810 T2285 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1038845 T2285 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1038853 T2285 oasc.Overseer.start Overseer (id=90221332246560771-127.0.0.1:39725_fi_zy-n_0000000000) starting
   [junit4]   2> 1038862 T2285 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1038874 T2309 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1038876 T2285 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1038885 T2285 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1038892 T2285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1038903 T2308 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1038905 T2310 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1038906 T2310 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1038909 T2310 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1040417 T2308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1040419 T2308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39725/fi_zy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39725_fi_zy",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1040420 T2308 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1040421 T2308 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1040433 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1040910 T2310 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1040911 T2310 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/collection1
   [junit4]   2> 1040911 T2310 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1040913 T2310 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1040914 T2310 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1040918 T2310 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/collection1/'
   [junit4]   2> 1040919 T2310 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/collection1/lib/README' to classloader
   [junit4]   2> 1040921 T2310 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/collection1/lib/classes/' to classloader
   [junit4]   2> 1040971 T2310 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1041007 T2310 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1041011 T2310 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1041018 T2310 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1041454 T2310 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1041455 T2310 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1041456 T2310 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1041465 T2310 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1041470 T2310 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1041507 T2310 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1041514 T2310 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1041521 T2310 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1041524 T2310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1041525 T2310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1041526 T2310 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1041529 T2310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1041530 T2310 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1041531 T2310 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1041532 T2310 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376668278412/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/control/data/
   [junit4]   2> 1041533 T2310 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@701a701a
   [junit4]   2> 1041534 T2310 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/control/data
   [junit4]   2> 1041534 T2310 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/control/data/index/
   [junit4]   2> 1041535 T2310 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1041536 T2310 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/control/data/index
   [junit4]   2> 1041536 T2310 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@14ce14ce lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50fb50fb),segFN=segments_1,generation=1}
   [junit4]   2> 1041537 T2310 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1041543 T2310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1041544 T2310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1041544 T2310 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1041545 T2310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1041545 T2310 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1041546 T2310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1041547 T2310 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1041547 T2310 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1041548 T2310 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1041549 T2310 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1041549 T2310 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1041550 T2310 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1041551 T2310 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1041551 T2310 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1041552 T2310 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1041554 T2310 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1041566 T2310 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1041566 T2310 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1041567 T2310 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@14ce14ce lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50fb50fb),segFN=segments_1,generation=1}
   [junit4]   2> 1041568 T2310 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1041569 T2310 oass.SolrIndexSearcher.<init> Opening Searcher@57965796 main
   [junit4]   2> 1041573 T2311 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57965796 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1041575 T2310 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1041575 T2310 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39725/fi_zy collection:control_collection shard:shard1
   [junit4]   2> 1041576 T2310 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1041584 T2310 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1041604 T2310 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1041611 T2310 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1041612 T2310 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1041612 T2310 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39725/fi_zy/collection1/
   [junit4]   2> 1041613 T2310 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1041613 T2310 oasc.SyncStrategy.syncToMe http://127.0.0.1:39725/fi_zy/collection1/ has no replicas
   [junit4]   2> 1041614 T2310 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39725/fi_zy/collection1/ shard1
   [junit4]   2> 1041615 T2310 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1041947 T2308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1041969 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1042059 T2310 oasc.ZkController.register We are http://127.0.0.1:39725/fi_zy/collection1/ and leader is http://127.0.0.1:39725/fi_zy/collection1/
   [junit4]   2> 1042060 T2310 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39725/fi_zy
   [junit4]   2> 1042060 T2310 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1042061 T2310 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1042061 T2310 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1042068 T2310 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1042071 T2285 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1042072 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1042073 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1042078 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 1042080 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1042082 T2314 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a0a3a0a name:ZooKeeperConnection Watcher:127.0.0.1:50098/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1042083 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1042086 T2285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1042093 T2285 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 1042265 T2285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1042267 T2285 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51163
   [junit4]   2> 1042268 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1042269 T2285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1042270 T2285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020
   [junit4]   2> 1042271 T2285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/'
   [junit4]   2> 1042283 T2285 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/solr.xml
   [junit4]   2> 1042358 T2285 oasc.CoreContainer.<init> New CoreContainer 554180872
   [junit4]   2> 1042359 T2285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/]
   [junit4]   2> 1042362 T2285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1042363 T2285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1042364 T2285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1042365 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1042366 T2285 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1042367 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1042368 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1042370 T2285 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1042371 T2285 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1042372 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1042380 T2285 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50098/solr
   [junit4]   2> 1042381 T2285 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1042382 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1042384 T2325 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7070707 name:ZooKeeperConnection Watcher:127.0.0.1:50098 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1042385 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1042390 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1042397 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1042399 T2327 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@238f238f name:ZooKeeperConnection Watcher:127.0.0.1:50098/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1042399 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1042413 T2285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1043424 T2285 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51163_fi_zy
   [junit4]   2> 1043427 T2285 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51163_fi_zy
   [junit4]   2> 1043433 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1043436 T2314 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1043438 T2327 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1043442 T2307 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1043463 T2328 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1043464 T2328 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1043466 T2328 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1043527 T2308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1043529 T2308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39725/fi_zy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39725_fi_zy",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1043545 T2308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51163/fi_zy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51163_fi_zy",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1043545 T2308 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1043546 T2308 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1043559 T2314 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1043559 T2327 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1043560 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1044467 T2328 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1044468 T2328 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/collection1
   [junit4]   2> 1044468 T2328 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1044470 T2328 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1044471 T2328 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1044475 T2328 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/collection1/'
   [junit4]   2> 1044476 T2328 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/collection1/lib/README' to classloader
   [junit4]   2> 1044477 T2328 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/collection1/lib/classes/' to classloader
   [junit4]   2> 1044543 T2328 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1044581 T2328 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1044586 T2328 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1044593 T2328 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1045037 T2328 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1045037 T2328 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1045038 T2328 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1045044 T2328 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1045047 T2328 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1045109 T2328 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1045117 T2328 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1045124 T2328 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1045129 T2328 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1045129 T2328 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1045130 T2328 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1045134 T2328 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1045135 T2328 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1045135 T2328 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1045136 T2328 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376668282020/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty1/
   [junit4]   2> 1045137 T2328 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@701a701a
   [junit4]   2> 1045138 T2328 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty1
   [junit4]   2> 1045138 T2328 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty1/index/
   [junit4]   2> 1045139 T2328 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1045140 T2328 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty1/index
   [junit4]   2> 1045140 T2328 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@c9e0c9e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ce20ce2),segFN=segments_1,generation=1}
   [junit4]   2> 1045141 T2328 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1045150 T2328 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1045151 T2328 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1045152 T2328 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1045153 T2328 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1045153 T2328 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1045154 T2328 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1045155 T2328 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1045156 T2328 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1045157 T2328 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1045159 T2328 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1045160 T2328 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1045161 T2328 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1045162 T2328 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1045164 T2328 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1045165 T2328 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1045168 T2328 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1045172 T2328 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1045173 T2328 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1045175 T2328 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@c9e0c9e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@ce20ce2),segFN=segments_1,generation=1}
   [junit4]   2> 1045176 T2328 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1045177 T2328 oass.SolrIndexSearcher.<init> Opening Searcher@603e603e main
   [junit4]   2> 1045184 T2329 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@603e603e main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1045190 T2328 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1045190 T2328 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51163/fi_zy collection:collection1 shard:shard1
   [junit4]   2> 1045191 T2328 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1045208 T2328 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1045233 T2328 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1045242 T2328 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1045242 T2328 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1045243 T2328 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:51163/fi_zy/collection1/
   [junit4]   2> 1045243 T2328 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1045244 T2328 oasc.SyncStrategy.syncToMe http://127.0.0.1:51163/fi_zy/collection1/ has no replicas
   [junit4]   2> 1045245 T2328 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:51163/fi_zy/collection1/ shard1
   [junit4]   2> 1045245 T2328 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1046598 T2308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1046648 T2314 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1046649 T2327 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1046650 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1046697 T2328 oasc.ZkController.register We are http://127.0.0.1:51163/fi_zy/collection1/ and leader is http://127.0.0.1:51163/fi_zy/collection1/
   [junit4]   2> 1046698 T2328 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51163/fi_zy
   [junit4]   2> 1046698 T2328 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1046699 T2328 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1046699 T2328 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1046705 T2328 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1046708 T2285 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1046709 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1046710 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1046906 T2285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1046908 T2285 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:40942
   [junit4]   2> 1046909 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1046910 T2285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1046910 T2285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642
   [junit4]   2> 1046911 T2285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/'
   [junit4]   2> 1046923 T2285 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/solr.xml
   [junit4]   2> 1046960 T2285 oasc.CoreContainer.<init> New CoreContainer 356914502
   [junit4]   2> 1046961 T2285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/]
   [junit4]   2> 1046962 T2285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1046963 T2285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1046964 T2285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1046964 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1046965 T2285 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1046966 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1046967 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1046967 T2285 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1046976 T2285 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1046977 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1046984 T2285 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50098/solr
   [junit4]   2> 1046985 T2285 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1046986 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1046989 T2341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6bbc6bbc name:ZooKeeperConnection Watcher:127.0.0.1:50098 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1046989 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1046993 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1047000 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1047002 T2343 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1fc11fc1 name:ZooKeeperConnection Watcher:127.0.0.1:50098/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1047003 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1047016 T2285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1048027 T2285 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40942_fi_zy
   [junit4]   2> 1048029 T2285 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40942_fi_zy
   [junit4]   2> 1048035 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1048036 T2327 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1048040 T2343 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1048041 T2314 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1048042 T2314 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1048048 T2307 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1048075 T2327 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1048087 T2344 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1048088 T2344 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1048091 T2344 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1048178 T2308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1048182 T2308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51163/fi_zy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:51163_fi_zy",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1048196 T2308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40942/fi_zy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:40942_fi_zy",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1048197 T2308 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1048197 T2308 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1048210 T2314 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1048210 T2327 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1048211 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1048212 T2343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1049092 T2344 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1049093 T2344 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/collection1
   [junit4]   2> 1049093 T2344 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1049095 T2344 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1049096 T2344 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1049100 T2344 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/collection1/'
   [junit4]   2> 1049101 T2344 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/collection1/lib/README' to classloader
   [junit4]   2> 1049102 T2344 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/collection1/lib/classes/' to classloader
   [junit4]   2> 1049148 T2344 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1049193 T2344 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1049197 T2344 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1049204 T2344 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1049655 T2344 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1049656 T2344 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1049656 T2344 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1049662 T2344 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1049666 T2344 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1049711 T2344 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1049718 T2344 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1049725 T2344 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1049729 T2344 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1049729 T2344 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1049730 T2344 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1049734 T2344 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1049734 T2344 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1049735 T2344 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1049735 T2344 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376668286642/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty2/
   [junit4]   2> 1049736 T2344 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@701a701a
   [junit4]   2> 1049737 T2344 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty2
   [junit4]   2> 1049738 T2344 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty2/index/
   [junit4]   2> 1049739 T2344 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1049739 T2344 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty2/index
   [junit4]   2> 1049740 T2344 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4a7e4a7e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bd32bd3),segFN=segments_1,generation=1}
   [junit4]   2> 1049741 T2344 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1049749 T2344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1049750 T2344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1049759 T2344 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1049760 T2344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1049760 T2344 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1049761 T2344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1049762 T2344 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1049762 T2344 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1049763 T2344 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1049764 T2344 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1049765 T2344 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1049765 T2344 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1049766 T2344 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1049767 T2344 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1049768 T2344 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1049769 T2344 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1049773 T2344 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1049773 T2344 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1049774 T2344 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@4a7e4a7e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2bd32bd3),segFN=segments_1,generation=1}
   [junit4]   2> 1049775 T2344 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1049776 T2344 oass.SolrIndexSearcher.<init> Opening Searcher@28182818 main
   [junit4]   2> 1049781 T2345 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@28182818 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1049785 T2344 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1049786 T2344 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:40942/fi_zy collection:collection1 shard:shard2
   [junit4]   2> 1049786 T2344 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1049794 T2344 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1049812 T2344 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1049820 T2344 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1049828 T2344 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1049829 T2344 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:40942/fi_zy/collection1/
   [junit4]   2> 1049829 T2344 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1049830 T2344 oasc.SyncStrategy.syncToMe http://127.0.0.1:40942/fi_zy/collection1/ has no replicas
   [junit4]   2> 1049830 T2344 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:40942/fi_zy/collection1/ shard2
   [junit4]   2> 1049831 T2344 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1051252 T2308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1051275 T2314 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1051275 T2343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1051276 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1051276 T2327 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1051372 T2344 oasc.ZkController.register We are http://127.0.0.1:40942/fi_zy/collection1/ and leader is http://127.0.0.1:40942/fi_zy/collection1/
   [junit4]   2> 1051373 T2344 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:40942/fi_zy
   [junit4]   2> 1051373 T2344 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1051374 T2344 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1051374 T2344 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1051378 T2344 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1051381 T2285 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1051382 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1051384 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1051590 T2285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1051592 T2285 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51590
   [junit4]   2> 1051593 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1051594 T2285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1051594 T2285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318
   [junit4]   2> 1051595 T2285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/'
   [junit4]   2> 1051607 T2285 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/solr.xml
   [junit4]   2> 1051656 T2285 oasc.CoreContainer.<init> New CoreContainer 1053310664
   [junit4]   2> 1051657 T2285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/]
   [junit4]   2> 1051660 T2285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1051661 T2285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1051662 T2285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1051663 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1051664 T2285 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1051665 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1051666 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1051667 T2285 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1051668 T2285 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1051669 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1051677 T2285 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:50098/solr
   [junit4]   2> 1051678 T2285 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1051680 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1051682 T2357 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ebb7ebb name:ZooKeeperConnection Watcher:127.0.0.1:50098 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1051683 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1051688 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1051698 T2285 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1051701 T2359 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ffb5ffb name:ZooKeeperConnection Watcher:127.0.0.1:50098/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1051702 T2285 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1051716 T2285 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1052728 T2285 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51590_fi_zy
   [junit4]   2> 1052731 T2285 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51590_fi_zy
   [junit4]   2> 1052739 T2327 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1052740 T2343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1052744 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1052745 T2314 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1052756 T2314 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1052757 T2359 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1052766 T2343 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1052767 T2327 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1052772 T2307 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1052784 T2360 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1052784 T2360 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1052787 T2360 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1052846 T2308 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1052849 T2308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40942/fi_zy",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:40942_fi_zy",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1052861 T2308 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51590/fi_zy",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "node_name":"127.0.0.1:51590_fi_zy",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1052862 T2308 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1052862 T2308 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1052874 T2307 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1052875 T2343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1052876 T2359 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1052875 T2314 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1052878 T2327 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1053788 T2360 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1053789 T2360 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/collection1
   [junit4]   2> 1053789 T2360 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1053792 T2360 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1053792 T2360 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1053798 T2360 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/collection1/'
   [junit4]   2> 1053800 T2360 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/collection1/lib/README' to classloader
   [junit4]   2> 1053802 T2360 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/collection1/lib/classes/' to classloader
   [junit4]   2> 1053881 T2360 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1053927 T2360 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1053931 T2360 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1053938 T2360 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1054407 T2360 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1054408 T2360 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1054409 T2360 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1054415 T2360 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1054418 T2360 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1054467 T2360 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1054475 T2360 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1054482 T2360 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1054486 T2360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1054487 T2360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1054487 T2360 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1054491 T2360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1054492 T2360 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1054493 T2360 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1054493 T2360 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376668291318/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty3/
   [junit4]   2> 1054494 T2360 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@701a701a
   [junit4]   2> 1054495 T2360 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty3
   [junit4]   2> 1054496 T2360 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty3/index/
   [junit4]   2> 1054496 T2360 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 1054497 T2360 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376668277684/jetty3/index
   [junit4]   2> 1054498 T2360 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@67986798 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3a603a60),segFN=segments_1,generation=1}
   [junit4]   2> 1054498 T2360 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1054515 T2360 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1054515 T2360 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1054516 T2360 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1054517 T2360 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1054517 T2360 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1054518 T2360 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1054519 T2360 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1054519 T2360 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1054520 T2360 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1054521 T2360 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1054522 T2360 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1054522 T2360 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1054523 T2360 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1054524 T2360 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1054525 T2360 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1054526 T2360 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1054530 T2360 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1054531 T2360 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1054532 T2360 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@67986798 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3a603a60),segFN=segments_1,generation=1}
   [junit4]   2> 1054532 T2360 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1054533 T2360 oass.SolrIndexSearcher.<init> Opening Searcher@664c664c main
   [junit4]   2> 1054539 T2361 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@664c664c main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1054543 T2360 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1054543 T2360 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51590/fi_zy collection:collection1 shard:shard1
   [junit4]   2> 1054544 T2360 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1054559 T2360 oasc.ZkController.register We are http://127.0.0.1:51590/fi_zy/collection1/ and leader is http://127.0.0.1:51163/fi_zy/collection1/
   [junit4]   2> 1054559 T2360 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51590/fi_zy
   [junit4]   2> 1054560 T2360 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 1054560 T2360 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C9704 name=collection1 org.apache.solr.core.SolrCore@270e270e url=http://127.0.0.1:51590/fi_zy/collection1 node=127.0.0.1:51590_fi_zy C9704_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:51590/fi_zy, state=down, node_name=127.0.0.1:51590_fi_zy, core=collection1}
   [junit4]   2> 1054561 T2362 C9704 P51590 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 1054562 T2362 C9704 P51590 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 1054563 T2362 C9704 P51590 oasc.ZkController.publish publishing core=collection1 state=recovering
   [junit4]   2> 1054563 T2362 C9704 P51590 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1054564 T2360 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1054566 T2362 C9704 P51590 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1054568 T2285 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1054569 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1054570 T2285 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1054582 T2320 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true
   [junit4]   2> 1054782 T2285 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1054784 T2285 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:59756
   [junit4]   2> 1054785 T2285 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1054786 T2285 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1054786 T2285 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376668294510
   [junit4]   2> 1054787 T2285 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376668294510/'
   [junit4]   2> 1054800 T2285 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376668294510/solr.xml
   [junit4]   2> 1054853 T2285 oasc.CoreContainer.<init> New CoreContainer 569188845
   [junit4]   2> 1054854 T2285 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376668294510/]
   [junit4]   2> 1054855 T2285 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1054856 T2285 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1054857 T2285 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1054858 T2285 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1054858 T2285 oashc.HttpShardHandlerFactory.getParameter

[...truncated too long message...]

ons/onenodecollection/leaders (1)
   [junit4]   1>      /solr/collections/onenodecollection/leaders/shard1 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "base_url":"http://127.0.0.1:51163/fi_zy",
   [junit4]   1>            "node_name":"127.0.0.1:51163_fi_zy",
   [junit4]   1>            "core":"onenodecollectioncore"}
   [junit4]   1>   /solr/aliases.json (0)
   [junit4]   1>   /solr/live_nodes (5)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:51163_fi_zy (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:40942_fi_zy (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:59756_fi_zy (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:39725_fi_zy (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:51590_fi_zy (0)
   [junit4]   1>   /solr/clusterstate.json (0)
   [junit4]   1>   DATA:
   [junit4]   1>       {
   [junit4]   1>         "control_collection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:39725/fi_zy",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:39725_fi_zy",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "onenodecollection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:51163/fi_zy",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:51163_fi_zy",
   [junit4]   1>                   "roles":"none",
   [junit4]   1>                   "core":"onenodecollectioncore",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "collection1":{
   [junit4]   1>           "shards":{
   [junit4]   1>             "shard1":{
   [junit4]   1>               "range":"80000000-ffffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:51163/fi_zy",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:51163_fi_zy",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node3":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:51590/fi_zy",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:51590_fi_zy",
   [junit4]   1>                   "core":"collection1"}}},
   [junit4]   1>             "shard2":{
   [junit4]   1>               "range":"0-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node2":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:40942/fi_zy",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:40942_fi_zy",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node4":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:59756/fi_zy",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:59756_fi_zy",
   [junit4]   1>                   "core":"collection1"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"}}
   [junit4]   1>  /zookeeper (1)
   [junit4]   1>  DATA:
   [junit4]   1>      
   [junit4]   1> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZk2Test -Dtests.method=testDistribSearch -Dtests.seed=3EE244D1B8580015 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_SY -Dtests.timezone=Africa/Tunis -Dtests.file.encoding=UTF-8
   [junit4] ERROR   33.3s J1 | BasicDistributedZk2Test.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:40942/fi_zy/onenodecollectioncore returned non ok status:404, message:Can not find: /fi_zy/onenodecollectioncore/update
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3EE244D1B8580015:BF04CAC9CF076029]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:738)
   [junit4]   2> 1071074 T2285 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 33345 T2284 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 1071901 T2376 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1071902 T2376 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene42: {}, docValues:{}, sim=DefaultSimilarity, locale=ar_SY, timezone=Africa/Tunis
   [junit4]   2> NOTE: Linux 3.2.0-51-generic x86/IBM Corporation 1.6.0 (32-bit)/cpus=8,threads=3,free=2374136,total=40013312
   [junit4]   2> NOTE: All tests run in this JVM: [TestCoreDiscovery, TestRTGBase, SpellingQueryConverterTest, NotRequiredUniqueKeyTest, TestFuzzyAnalyzedSuggestions, BasicZkTest, DirectSolrConnectionTest, UniqFieldsUpdateProcessorFactoryTest, SpellCheckComponentTest, TestRandomFaceting, TestNumberUtils, PluginInfoTest, DocumentAnalysisRequestHandlerTest, TestSolr4Spatial, TestIndexSearcher, SolrCoreCheckLockOnStartupTest, PreAnalyzedFieldTest, TestCodecSupport, TimeZoneUtilsTest, TestPhraseSuggestions, StatsComponentTest, DisMaxRequestHandlerTest, SuggesterTSTTest, TestRangeQuery, SolrPluginUtilsTest, BlockCacheTest, TestSolrCoreProperties, PingRequestHandlerTest, CollectionsAPIDistributedZkTest, TestQueryTypes, DebugComponentTest, TestFastLRUCache, PrimUtilsTest, TestFieldResource, TestCloudManagedSchemaAddField, HighlighterConfigTest, SpellCheckCollatorTest, QueryResultKeyTest, TestFoldingMultitermQuery, TestXIncludeConfig, TestSolrJ, ChaosMonkeyNothingIsSafeTest, CoreContainerCoreInitFailuresTest, DirectSolrSpellCheckerTest, HdfsLockFactoryTest, TestRecovery, TestDefaultSimilarityFactory, TestPartialUpdateDeduplication, TestUtils, TestFiltering, RequiredFieldsTest, TestSweetSpotSimilarityFactory, TestStressRecovery, AliasIntegrationTest, BadCopyFieldTest, CurrencyFieldOpenExchangeTest, ZkNodePropsTest, SignatureUpdateProcessorFactoryTest, TestAddFieldRealTimeGet, FileUtilsTest, BadIndexSchemaTest, DocValuesMultiTest, HighlighterTest, TestIndexingPerformance, LeaderElectionIntegrationTest, StandardRequestHandlerTest, TestQuerySenderListener, DefaultValueUpdateProcessorTest, IndexSchemaRuntimeFieldTest, OpenCloseCoreStressTest, TestCollationField, PathHierarchyTokenizerFactoryTest, NoCacheHeaderTest, TestDynamicFieldCollectionResource, IndexBasedSpellCheckerTest, TestUpdate, TestReload, SimplePostToolTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test]
   [junit4] Completed on J1 in 34.38s, 1 test, 1 error <<< FAILURES!

[...truncated 495 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:395: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:388: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:181: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:450: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1233: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:876: There were test failures: 321 suites, 1365 tests, 1 error, 34 ignored (6 assumptions)

Total time: 54 minutes 19 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/ibm-j9-jdk6 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure