You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/24 02:12:31 UTC

[JENKINS] Lucene-Solr-trunk-Linux (32bit/ibm-j9-jdk7) - Build # 7142 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-trunk-Linux/7142/
Java: 32bit/ibm-j9-jdk7 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}

1 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
Server at http://127.0.0.1:57571/onenodecollectioncore returned non ok status:404, message:Can not find: /onenodecollectioncore/update

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:57571/onenodecollectioncore returned non ok status:404, message:Can not find: /onenodecollectioncore/update
	at __randomizedtesting.SeedInfo.seed([3BD54DBC83439D73:BA33C3A4F41CFD4F]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:88)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:55)
	at java.lang.reflect.Method.invoke(Method.java:613)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:780)




Build Log:
[...truncated 9940 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZk2Test
   [junit4]   2> 601520 T2136 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 601526 T2136 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZk2Test-1377302421454
   [junit4]   2> 601528 T2136 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 601528 T2137 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 601629 T2136 oasc.ZkTestServer.run start zk server on port:35542
   [junit4]   2> 601631 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 601642 T2143 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e020ed9 name:ZooKeeperConnection Watcher:127.0.0.1:35542 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 601642 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 601643 T2136 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 601646 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 601647 T2145 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24206966 name:ZooKeeperConnection Watcher:127.0.0.1:35542/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 601648 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 601648 T2136 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 601650 T2136 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 601652 T2136 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 601654 T2136 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 601656 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 601656 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 601659 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 601660 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 601662 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 601663 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 601665 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 601666 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 601668 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 601669 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 601672 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 601672 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 601675 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 601676 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 601677 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 601678 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 601680 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 601681 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 601683 T2136 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 601683 T2136 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 601765 T2136 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 601770 T2136 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:55572
   [junit4]   2> 601770 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 601771 T2136 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 601772 T2136 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613
   [junit4]   2> 601772 T2136 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/'
   [junit4]   2> 601781 T2136 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/solr.xml
   [junit4]   2> 601820 T2136 oasc.CoreContainer.<init> New CoreContainer 1998348826
   [junit4]   2> 601821 T2136 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/]
   [junit4]   2> 601823 T2136 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 601824 T2136 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 601825 T2136 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 601826 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 601827 T2136 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 601828 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 601829 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 601830 T2136 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 601830 T2136 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 601831 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 601839 T2136 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35542/solr
   [junit4]   2> 601840 T2136 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 601842 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 601845 T2156 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a6ce8db4 name:ZooKeeperConnection Watcher:127.0.0.1:35542 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 601845 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 601847 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 601857 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 601858 T2158 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@23fe752 name:ZooKeeperConnection Watcher:127.0.0.1:35542/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 601858 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 601860 T2136 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 601863 T2136 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 601865 T2136 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 601867 T2136 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55572_
   [junit4]   2> 601868 T2136 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55572_
   [junit4]   2> 601872 T2136 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 601877 T2136 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 601879 T2136 oasc.Overseer.start Overseer (id=90262891492671491-127.0.0.1:55572_-n_0000000000) starting
   [junit4]   2> 601881 T2136 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 601884 T2160 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 601884 T2136 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 601886 T2136 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 601888 T2136 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 601890 T2159 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 601892 T2161 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 601893 T2161 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 601894 T2161 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 603394 T2159 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 603395 T2159 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55572",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:55572_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 603396 T2159 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 603396 T2159 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 603401 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 603894 T2161 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 603895 T2161 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/collection1
   [junit4]   2> 603895 T2161 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 603896 T2161 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 603896 T2161 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 603898 T2161 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/collection1/'
   [junit4]   2> 603898 T2161 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/collection1/lib/classes/' to classloader
   [junit4]   2> 603899 T2161 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/collection1/lib/README' to classloader
   [junit4]   2> 603928 T2161 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 603959 T2161 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 603960 T2161 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 603966 T2161 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 604284 T2161 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 604289 T2161 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 604291 T2161 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 604303 T2161 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 604307 T2161 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 604311 T2161 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 604312 T2161 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 604313 T2161 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 604313 T2161 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 604315 T2161 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 604315 T2161 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 604316 T2161 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 604316 T2161 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1377302421613/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/control/data/
   [junit4]   2> 604317 T2161 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@57b0c119
   [junit4]   2> 604318 T2161 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/control/data
   [junit4]   2> 604319 T2161 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/control/data/index/
   [junit4]   2> 604319 T2161 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 604320 T2161 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/control/data/index
   [junit4]   2> 604321 T2161 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 604322 T2161 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8f1a5999 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e7ff3073),segFN=segments_1,generation=1}
   [junit4]   2> 604323 T2161 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 604325 T2161 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 604326 T2161 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 604326 T2161 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 604326 T2161 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 604327 T2161 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 604327 T2161 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 604328 T2161 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 604328 T2161 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 604328 T2161 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 604329 T2161 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 604329 T2161 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 604330 T2161 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 604330 T2161 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 604330 T2161 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 604331 T2161 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 604332 T2161 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 604335 T2161 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 604335 T2161 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 604335 T2161 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 604336 T2161 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8f1a5999 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@e7ff3073),segFN=segments_1,generation=1}
   [junit4]   2> 604336 T2161 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 604337 T2161 oass.SolrIndexSearcher.<init> Opening Searcher@615c154f main
   [junit4]   2> 604339 T2162 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@615c154f main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 604339 T2161 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 604339 T2161 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55572 collection:control_collection shard:shard1
   [junit4]   2> 604340 T2161 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 604344 T2161 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 604349 T2161 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 604351 T2161 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 604351 T2161 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 604352 T2161 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55572/collection1/
   [junit4]   2> 604352 T2161 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 604352 T2161 oasc.SyncStrategy.syncToMe http://127.0.0.1:55572/collection1/ has no replicas
   [junit4]   2> 604353 T2161 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55572/collection1/ shard1
   [junit4]   2> 604353 T2161 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 604906 T2159 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 604913 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 604959 T2161 oasc.ZkController.register We are http://127.0.0.1:55572/collection1/ and leader is http://127.0.0.1:55572/collection1/
   [junit4]   2> 604960 T2161 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55572
   [junit4]   2> 604960 T2161 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 604960 T2161 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 604960 T2161 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 604963 T2161 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 604964 T2136 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 604964 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 604965 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 604969 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 604971 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 604973 T2165 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d7eb7da1 name:ZooKeeperConnection Watcher:127.0.0.1:35542/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 604973 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 604974 T2136 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 604976 T2136 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 605041 T2136 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 605045 T2136 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:39727
   [junit4]   2> 605046 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 605046 T2136 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 605047 T2136 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904
   [junit4]   2> 605048 T2136 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/'
   [junit4]   2> 605055 T2136 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/solr.xml
   [junit4]   2> 605086 T2136 oasc.CoreContainer.<init> New CoreContainer -1321340903
   [junit4]   2> 605087 T2136 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/]
   [junit4]   2> 605088 T2136 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 605088 T2136 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 605089 T2136 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 605090 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 605090 T2136 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 605091 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 605091 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 605092 T2136 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 605093 T2136 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 605093 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 605098 T2136 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35542/solr
   [junit4]   2> 605098 T2136 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 605100 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 605101 T2176 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@228043de name:ZooKeeperConnection Watcher:127.0.0.1:35542 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 605102 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 605103 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 605109 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 605110 T2178 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b323213 name:ZooKeeperConnection Watcher:127.0.0.1:35542/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 605111 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 605114 T2136 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 606116 T2136 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39727_
   [junit4]   2> 606118 T2136 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39727_
   [junit4]   2> 606120 T2165 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 606121 T2178 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 606120 T2158 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 606121 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 606126 T2179 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 606126 T2179 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 606127 T2179 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 606419 T2159 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 606420 T2159 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55572",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:55572_",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 606425 T2159 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39727",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39727_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 606426 T2159 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 606427 T2159 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 606432 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 606432 T2165 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 606432 T2178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 607128 T2179 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 607129 T2179 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/collection1
   [junit4]   2> 607130 T2179 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 607131 T2179 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 607132 T2179 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 607135 T2179 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/collection1/'
   [junit4]   2> 607137 T2179 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/collection1/lib/classes/' to classloader
   [junit4]   2> 607138 T2179 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/collection1/lib/README' to classloader
   [junit4]   2> 607198 T2179 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 607233 T2179 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 607234 T2179 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 607240 T2179 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 607588 T2179 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 607593 T2179 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 607595 T2179 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 607604 T2179 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 607607 T2179 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 607611 T2179 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 607611 T2179 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 607612 T2179 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 607612 T2179 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 607613 T2179 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 607613 T2179 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 607614 T2179 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 607614 T2179 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1377302424904/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty1/
   [junit4]   2> 607614 T2179 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@57b0c119
   [junit4]   2> 607615 T2179 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty1
   [junit4]   2> 607616 T2179 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty1/index/
   [junit4]   2> 607616 T2179 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 607616 T2179 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty1/index
   [junit4]   2> 607617 T2179 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 607617 T2179 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2c8e5ed1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9627f6e2),segFN=segments_1,generation=1}
   [junit4]   2> 607618 T2179 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 607620 T2179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 607620 T2179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 607621 T2179 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 607621 T2179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 607621 T2179 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 607622 T2179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 607622 T2179 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 607622 T2179 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 607623 T2179 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 607623 T2179 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 607624 T2179 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 607624 T2179 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 607625 T2179 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 607625 T2179 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 607625 T2179 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 607627 T2179 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 607629 T2179 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 607629 T2179 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 607630 T2179 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 607631 T2179 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@2c8e5ed1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@9627f6e2),segFN=segments_1,generation=1}
   [junit4]   2> 607631 T2179 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 607631 T2179 oass.SolrIndexSearcher.<init> Opening Searcher@c5d4138 main
   [junit4]   2> 607634 T2180 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c5d4138 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 607635 T2179 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 607635 T2179 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39727 collection:collection1 shard:shard1
   [junit4]   2> 607636 T2179 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 607640 T2179 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 607646 T2179 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 607648 T2179 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 607648 T2179 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 607648 T2179 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39727/collection1/
   [junit4]   2> 607649 T2179 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 607649 T2179 oasc.SyncStrategy.syncToMe http://127.0.0.1:39727/collection1/ has no replicas
   [junit4]   2> 607650 T2179 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39727/collection1/ shard1
   [junit4]   2> 607650 T2179 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 607939 T2159 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 607944 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 607944 T2178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 607944 T2165 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 607956 T2179 oasc.ZkController.register We are http://127.0.0.1:39727/collection1/ and leader is http://127.0.0.1:39727/collection1/
   [junit4]   2> 607956 T2179 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39727
   [junit4]   2> 607957 T2179 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 607957 T2179 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 607957 T2179 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 607960 T2179 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 607961 T2136 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 607961 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 607962 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 608033 T2136 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 608036 T2136 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:57571
   [junit4]   2> 608037 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 608038 T2136 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 608038 T2136 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893
   [junit4]   2> 608039 T2136 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/'
   [junit4]   2> 608047 T2136 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/solr.xml
   [junit4]   2> 608078 T2136 oasc.CoreContainer.<init> New CoreContainer 1275150559
   [junit4]   2> 608079 T2136 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/]
   [junit4]   2> 608080 T2136 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 608081 T2136 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 608081 T2136 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 608082 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 608082 T2136 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 608083 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 608083 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 608084 T2136 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 608085 T2136 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 608086 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 608090 T2136 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35542/solr
   [junit4]   2> 608091 T2136 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 608092 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 608094 T2192 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@af8e832f name:ZooKeeperConnection Watcher:127.0.0.1:35542 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 608094 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 608096 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 608101 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 608103 T2194 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4395a06a name:ZooKeeperConnection Watcher:127.0.0.1:35542/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 608103 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 608107 T2136 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 609110 T2136 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:57571_
   [junit4]   2> 609111 T2136 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:57571_
   [junit4]   2> 609113 T2165 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 609113 T2178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 609114 T2194 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 609116 T2165 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 609116 T2158 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 609117 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 609120 T2178 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 609125 T2195 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 609126 T2195 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 609127 T2195 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 609447 T2159 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 609448 T2159 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39727",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39727_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 609450 T2159 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57571",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:57571_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 609451 T2159 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 609451 T2159 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 609454 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 609454 T2165 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 609454 T2194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 609454 T2178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 610128 T2195 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 610128 T2195 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/collection1
   [junit4]   2> 610129 T2195 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 610130 T2195 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 610130 T2195 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 610131 T2195 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/collection1/'
   [junit4]   2> 610132 T2195 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/collection1/lib/classes/' to classloader
   [junit4]   2> 610133 T2195 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/collection1/lib/README' to classloader
   [junit4]   2> 610178 T2195 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 610227 T2195 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 610228 T2195 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 610234 T2195 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 610579 T2195 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 610584 T2195 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 610587 T2195 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 610599 T2195 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 610603 T2195 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 610608 T2195 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 610609 T2195 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 610610 T2195 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 610610 T2195 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 610611 T2195 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 610612 T2195 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 610612 T2195 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 610612 T2195 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1377302427893/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty2/
   [junit4]   2> 610613 T2195 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@57b0c119
   [junit4]   2> 610614 T2195 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty2
   [junit4]   2> 610615 T2195 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty2/index/
   [junit4]   2> 610615 T2195 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 610616 T2195 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty2/index
   [junit4]   2> 610616 T2195 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 610617 T2195 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8ce1d290 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@aa02126),segFN=segments_1,generation=1}
   [junit4]   2> 610618 T2195 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 610620 T2195 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 610621 T2195 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 610621 T2195 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 610622 T2195 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 610622 T2195 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 610623 T2195 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 610624 T2195 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 610624 T2195 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 610625 T2195 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 610625 T2195 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 610626 T2195 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 610627 T2195 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 610627 T2195 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 610628 T2195 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 610629 T2195 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 610630 T2195 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 610632 T2195 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 610633 T2195 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 610633 T2195 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 610634 T2195 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@8ce1d290 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@aa02126),segFN=segments_1,generation=1}
   [junit4]   2> 610634 T2195 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 610635 T2195 oass.SolrIndexSearcher.<init> Opening Searcher@c2532de main
   [junit4]   2> 610637 T2196 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@c2532de main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 610639 T2195 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 610639 T2195 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:57571 collection:collection1 shard:shard2
   [junit4]   2> 610639 T2195 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 610644 T2195 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 610649 T2195 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 610651 T2195 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 610651 T2195 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 610652 T2195 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:57571/collection1/
   [junit4]   2> 610652 T2195 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 610652 T2195 oasc.SyncStrategy.syncToMe http://127.0.0.1:57571/collection1/ has no replicas
   [junit4]   2> 610653 T2195 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:57571/collection1/ shard2
   [junit4]   2> 610653 T2195 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 610960 T2159 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 610970 T2165 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 610970 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 610970 T2178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 610970 T2194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 611008 T2195 oasc.ZkController.register We are http://127.0.0.1:57571/collection1/ and leader is http://127.0.0.1:57571/collection1/
   [junit4]   2> 611009 T2195 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:57571
   [junit4]   2> 611010 T2195 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 611010 T2195 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 611011 T2195 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 611013 T2195 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 611015 T2136 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 611017 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 611018 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 611114 T2136 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 611117 T2136 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51596
   [junit4]   2> 611118 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 611119 T2136 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 611119 T2136 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954
   [junit4]   2> 611120 T2136 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/'
   [junit4]   2> 611128 T2136 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/solr.xml
   [junit4]   2> 611159 T2136 oasc.CoreContainer.<init> New CoreContainer 1260409062
   [junit4]   2> 611160 T2136 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/]
   [junit4]   2> 611162 T2136 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 611162 T2136 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 611163 T2136 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 611163 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 611164 T2136 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 611165 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 611165 T2136 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 611166 T2136 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 611166 T2136 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 611167 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 611172 T2136 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35542/solr
   [junit4]   2> 611172 T2136 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 611174 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 611175 T2208 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ee4f1d9 name:ZooKeeperConnection Watcher:127.0.0.1:35542 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 611175 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 611177 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 611182 T2136 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 611183 T2210 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8d45299d name:ZooKeeperConnection Watcher:127.0.0.1:35542/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 611183 T2136 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 611186 T2136 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 612189 T2136 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51596_
   [junit4]   2> 612191 T2136 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51596_
   [junit4]   2> 612195 T2165 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 612195 T2178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 612197 T2158 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 612197 T2194 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 612198 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 612199 T2178 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 612199 T2194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 612197 T2210 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 612199 T2165 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 612207 T2211 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 612208 T2211 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 612210 T2211 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 612477 T2159 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 612478 T2159 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:57571",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:57571_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 612483 T2159 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:51596",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "node_name":"127.0.0.1:51596_",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 612484 T2159 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 612484 T2159 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 612490 T2165 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 612490 T2178 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 612490 T2210 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 612490 T2158 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 612490 T2194 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 613210 T2211 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 613211 T2211 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/collection1
   [junit4]   2> 613212 T2211 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 613213 T2211 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 613214 T2211 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 613216 T2211 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/collection1/'
   [junit4]   2> 613217 T2211 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/collection1/lib/classes/' to classloader
   [junit4]   2> 613218 T2211 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/collection1/lib/README' to classloader
   [junit4]   2> 613273 T2211 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 613304 T2211 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 613305 T2211 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 613311 T2211 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 613652 T2211 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 613657 T2211 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 613660 T2211 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 613669 T2211 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 613673 T2211 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 613677 T2211 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 613678 T2211 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 613679 T2211 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 613679 T2211 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 613680 T2211 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 613680 T2211 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 613681 T2211 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 613681 T2211 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1377302430954/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty3/
   [junit4]   2> 613682 T2211 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@57b0c119
   [junit4]   2> 613682 T2211 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty3
   [junit4]   2> 613683 T2211 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty3/index/
   [junit4]   2> 613683 T2211 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 613684 T2211 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1377302421454/jetty3/index
   [junit4]   2> 613684 T2211 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=32, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 613685 T2211 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@cb0b67e8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4f9f61d5),segFN=segments_1,generation=1}
   [junit4]   2> 613685 T2211 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 613688 T2211 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 613688 T2211 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 613689 T2211 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 613689 T2211 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 613690 T2211 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 613690 T2211 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 613691 T2211 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 613691 T2211 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 613691 T2211 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 613692 T2211 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 613693 T2211 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 613693 T2211 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 613693 T2211 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 613694 T2211 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 613695 T2211 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 613696 T2211 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 613699 T2211 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 613699 T2211 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 613700 T2211 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=45, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 613700 T2211 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@cb0b67e8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4f9f61d5),segFN=segments_1,generation=1}
   [junit4]   2> 613701 T2211 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 613701 T2211 oass.SolrIndexSearcher.<init> Opening Searcher@8df0425f main
   [junit4]   2> 613704 T2212 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@8df0425f main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 613705 T2211 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 613705 T2211 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:51596 collection:collection1 shard:shard1
   [junit4]   2> 613706 T2211 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 613712 T2211 oasc.ZkController.register We are http://127.0.0.1:51596/collection1/ and leader is http://127.0.0.1:39727/collection1/
   [junit4]   2> 613712 T2211 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:51596
   [junit4]   2> 613712 T2211 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 613712 T2211 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C530 name=collection1 org.apache.solr.core.SolrCore@d0bf53de url=http://127.0.0.1:51596/collection1 node=127.0.0.1:51596_ C530_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:51596, state=down, node_name=127.0.0.1:51596_, core=collection1}
   [junit4]   2> 613713 T2213 C530 P51596 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 613713 T2211 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 613713 T2213 C530 P51596 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 613714 T2213 C530 P51596 oasc.ZkController.publish publishing core=collection1 state=recovering
   [junit4]   2> 613714 T2213 C530 P51596 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 613714 T2136 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 613715 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 613715 T2213 C530 P51596 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 613716 T2136 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 613727 T2171 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true
   [junit4]   2> 613791 T2136 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 613794 T2136 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:37270
   [junit4]   2> 613795 T2136 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 613796 T2136 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 613796 T2136 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ..

[...truncated too long message...]

  "router":"compositeId",
   [junit4]   1>          "configName":"conf1"}
   [junit4]   1>     /solr/collections/onenodecollection/leader_elect (1)
   [junit4]   1>      /solr/collections/onenodecollection/leader_elect/shard1 (1)
   [junit4]   1>       /solr/collections/onenodecollection/leader_elect/shard1/election (1)
   [junit4]   1>        /solr/collections/onenodecollection/leader_elect/shard1/election/90262891492671494-core_node1-n_0000000000 (0)
   [junit4]   1>     /solr/collections/onenodecollection/leaders (1)
   [junit4]   1>      /solr/collections/onenodecollection/leaders/shard1 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "base_url":"http://127.0.0.1:39727",
   [junit4]   1>            "node_name":"127.0.0.1:39727_",
   [junit4]   1>            "core":"onenodecollectioncore"}
   [junit4]   1>   /solr/aliases.json (0)
   [junit4]   1>   /solr/live_nodes (5)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:55572_ (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:37270_ (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:51596_ (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:39727_ (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:57571_ (0)
   [junit4]   1>   /solr/clusterstate.json (0)
   [junit4]   1>   DATA:
   [junit4]   1>       {
   [junit4]   1>         "control_collection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:55572",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:55572_",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "onenodecollection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:39727",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:39727_",
   [junit4]   1>                   "roles":"none",
   [junit4]   1>                   "core":"onenodecollectioncore",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "collection1":{
   [junit4]   1>           "shards":{
   [junit4]   1>             "shard1":{
   [junit4]   1>               "range":"80000000-ffffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:39727",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:39727_",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node3":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:51596",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:51596_",
   [junit4]   1>                   "core":"collection1"}}},
   [junit4]   1>             "shard2":{
   [junit4]   1>               "range":"0-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node2":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:57571",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:57571_",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node4":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:37270",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:37270_",
   [junit4]   1>                   "core":"collection1"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"}}
   [junit4]   1>  /zookeeper (1)
   [junit4]   1>  DATA:
   [junit4]   1>      
   [junit4]   1> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZk2Test -Dtests.method=testDistribSearch -Dtests.seed=3BD54DBC83439D73 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=pt_PT -Dtests.timezone=Pacific/Wallis -Dtests.file.encoding=UTF-8
   [junit4] ERROR   28.2s J0 | BasicDistributedZk2Test.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:57571/onenodecollectioncore returned non ok status:404, message:Can not find: /onenodecollectioncore/update
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([3BD54DBC83439D73:BA33C3A4F41CFD4F]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:780)
   [junit4]   2> 629685 T2136 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 28169 T2135 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=pt_PT, timezone=Pacific/Wallis
   [junit4]   2> NOTE: Linux 3.2.0-52-generic x86/IBM Corporation 1.7.0 (32-bit)/cpus=8,threads=2,free=118267168,total=257294336
   [junit4]   2> NOTE: All tests run in this JVM: [BlockDirectoryTest, TestLuceneMatchVersion, TestFieldTypeResource, DateMathParserTest, TestCoreContainer, TestJoin, HdfsLockFactoryTest, JsonLoaderTest, BasicDistributedZkTest, ResourceLoaderTest, TestSolrXMLSerializer, QueryEqualityTest, SliceStateTest, SolrXmlInZkTest, TestConfig, TestCloudManagedSchema, ClusterStateUpdateTest, TestSystemIdResolver, FieldMutatingUpdateProcessorTest, TestElisionMultitermQuery, UUIDFieldTest, TestFaceting, SyncSliceTest, DocumentBuilderTest, HdfsRecoveryZkTest, TestJmxMonitoredMap, BJQParserTest, TermVectorComponentTest, TestDocumentBuilder, TestPseudoReturnFields, TestTrie, TestSolrQueryParser, TestSchemaSimilarityResource, URLClassifyProcessorTest, TestPHPSerializedResponseWriter, BadComponentTest, StatelessScriptUpdateProcessorFactoryTest, AnalysisAfterCoreReloadTest, TestCharFilters, TestSort, TestRemoteStreaming, SoftAutoCommitTest, SystemInfoHandlerTest, HdfsCollectionsAPIDistributedZkTest, ClusterStateTest, LukeRequestHandlerTest, TestImplicitCoreProperties, LegacyHTMLStripCharFilterTest, LoggingHandlerTest, TestPerFieldSimilarity, SearchHandlerTest, BadCopyFieldTest, TestSuggestSpellingConverter, TermsComponentTest, TestDocSet, RecoveryZkTest, CopyFieldTest, FastVectorHighlighterTest, BinaryUpdateRequestHandlerTest, TestBM25SimilarityFactory, MoreLikeThisHandlerTest, PolyFieldTest, BlockCacheTest, TestClassNameShortening, TestBadConfig, TestMaxScoreQueryParser, TestReplicationHandler, ZkCLITest, ReturnFieldsTest, ExternalFileFieldSortTest, TestLMJelinekMercerSimilarityFactory, AutoCommitTest, DOMUtilTest, SliceStateUpdateTest, TestLFUCache, SimpleFacetsTest, SOLR749Test, BasicDistributedZk2Test]
   [junit4] Completed on J0 in 28.70s, 1 test, 1 error <<< FAILURES!

[...truncated 523 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:389: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:382: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/build.xml:181: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/solr/common-build.xml:450: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:1230: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-trunk-Linux/lucene/common-build.xml:873: There were test failures: 324 suites, 1405 tests, 1 error, 35 ignored (6 assumptions)

Total time: 44 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/ibm-j9-jdk7 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure