You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2013/08/12 14:11:36 UTC

[JENKINS] Lucene-Solr-4.x-Linux (32bit/ibm-j9-jdk6) - Build # 6900 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/6900/
Java: 32bit/ibm-j9-jdk6 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}

1 tests failed.
FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
Server at http://127.0.0.1:33314/nl_c/zk/onenodecollectioncore returned non ok status:404, message:Can not find: /nl_c/zk/onenodecollectioncore/update

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:33314/nl_c/zk/onenodecollectioncore returned non ok status:404, message:Can not find: /nl_c/zk/onenodecollectioncore/update
	at __randomizedtesting.SeedInfo.seed([762003F01ED3573D:F7C68DE8698C3701]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:60)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:37)
	at java.lang.reflect.Method.invoke(Method.java:611)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:738)




Build Log:
[...truncated 9336 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZk2Test
   [junit4]   2> 1021711 T1873 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /nl_c/zk
   [junit4]   2> 1021714 T1873 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZk2Test-1376308649312
   [junit4]   2> 1021716 T1873 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1021717 T1874 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1021817 T1873 oasc.ZkTestServer.run start zk server on port:41127
   [junit4]   2> 1021818 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1021885 T1880 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21e121e1 name:ZooKeeperConnection Watcher:127.0.0.1:41127 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1021885 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1021886 T1873 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1021893 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1021906 T1882 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15ef15ef name:ZooKeeperConnection Watcher:127.0.0.1:41127/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1021907 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1021908 T1873 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1021915 T1873 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1021922 T1873 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1021928 T1873 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1021936 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1021937 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1021948 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1021949 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1021957 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1021958 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1021965 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1021966 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1021974 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1021975 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1021983 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1021984 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1021992 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1021993 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1022001 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1022002 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1022009 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1022010 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1022018 T1873 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1022035 T1873 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1022219 T1873 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1022223 T1873 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:45558
   [junit4]   2> 1022224 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1022224 T1873 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1022225 T1873 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643
   [junit4]   2> 1022226 T1873 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/'
   [junit4]   2> 1022238 T1873 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/solr.xml
   [junit4]   2> 1022285 T1873 oasc.CoreContainer.<init> New CoreContainer 51577619
   [junit4]   2> 1022286 T1873 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/]
   [junit4]   2> 1022287 T1873 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1022288 T1873 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1022289 T1873 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1022289 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1022290 T1873 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1022291 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1022292 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1022292 T1873 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1022293 T1873 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1022294 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1022302 T1873 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41127/solr
   [junit4]   2> 1022303 T1873 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1022304 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1022307 T1893 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@309a309a name:ZooKeeperConnection Watcher:127.0.0.1:41127 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1022308 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1022312 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1022320 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1022322 T1895 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6da86da8 name:ZooKeeperConnection Watcher:127.0.0.1:41127/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1022323 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1022327 T1873 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1022337 T1873 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1022356 T1873 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1022362 T1873 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45558_nl_c%2Fzk
   [junit4]   2> 1022365 T1873 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45558_nl_c%2Fzk
   [junit4]   2> 1022375 T1873 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1022399 T1873 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1022406 T1873 oasc.Overseer.start Overseer (id=90197763641573379-127.0.0.1:45558_nl_c%2Fzk-n_0000000000) starting
   [junit4]   2> 1022416 T1873 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1022428 T1897 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1022431 T1873 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1022440 T1873 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1022446 T1873 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1022457 T1896 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1022460 T1898 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1022460 T1898 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1022463 T1898 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1023972 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1023976 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45558/nl_c/zk",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45558_nl_c%2Fzk",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1023977 T1896 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1023977 T1896 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1023991 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1024464 T1898 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1024465 T1898 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/collection1
   [junit4]   2> 1024465 T1898 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1024468 T1898 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1024468 T1898 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1024473 T1898 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/collection1/'
   [junit4]   2> 1024475 T1898 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/collection1/lib/README' to classloader
   [junit4]   2> 1024475 T1898 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/collection1/lib/classes/' to classloader
   [junit4]   2> 1024511 T1898 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1024554 T1898 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1024575 T1898 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1024582 T1898 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1025007 T1898 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1025008 T1898 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1025009 T1898 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1025026 T1898 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1025030 T1898 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1025065 T1898 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1025072 T1898 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1025079 T1898 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1025082 T1898 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1025083 T1898 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1025084 T1898 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1025087 T1898 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1025088 T1898 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1025089 T1898 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1025089 T1898 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1376308649643/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/control/data/
   [junit4]   2> 1025090 T1898 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e475e47
   [junit4]   2> 1025091 T1898 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/control/data
   [junit4]   2> 1025092 T1898 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/control/data/index/
   [junit4]   2> 1025092 T1898 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1025093 T1898 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/control/data/index
   [junit4]   2> 1025096 T1898 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@12021202 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4f954f95),segFN=segments_1,generation=1}
   [junit4]   2> 1025096 T1898 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1025103 T1898 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1025104 T1898 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1025104 T1898 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1025105 T1898 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1025106 T1898 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1025107 T1898 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1025108 T1898 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1025108 T1898 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1025119 T1898 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1025120 T1898 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1025123 T1898 oass.SolrIndexSearcher.<init> Opening Searcher@326e326e main
   [junit4]   2> 1025124 T1898 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1025125 T1898 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1025130 T1899 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@326e326e main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 1025134 T1898 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1025134 T1898 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45558/nl_c/zk collection:control_collection shard:shard1
   [junit4]   2> 1025135 T1898 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1025143 T1898 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1025163 T1898 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1025170 T1898 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1025171 T1898 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1025172 T1898 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45558/nl_c/zk/collection1/
   [junit4]   2> 1025173 T1898 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1025174 T1898 oasc.SyncStrategy.syncToMe http://127.0.0.1:45558/nl_c/zk/collection1/ has no replicas
   [junit4]   2> 1025175 T1898 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45558/nl_c/zk/collection1/ shard1
   [junit4]   2> 1025175 T1898 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1025546 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1025603 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1025645 T1898 oasc.ZkController.register We are http://127.0.0.1:45558/nl_c/zk/collection1/ and leader is http://127.0.0.1:45558/nl_c/zk/collection1/
   [junit4]   2> 1025646 T1898 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45558/nl_c/zk
   [junit4]   2> 1025647 T1898 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1025647 T1898 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1025648 T1898 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1025656 T1898 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1025659 T1873 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 1025660 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1025661 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1025667 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:
   [junit4]   2> 1025668 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1025672 T1902 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@68f068f name:ZooKeeperConnection Watcher:127.0.0.1:41127/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1025672 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1025676 T1873 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1025684 T1873 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:true cause connection loss:true
   [junit4]   2> 1025886 T1873 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1025889 T1873 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:60883
   [junit4]   2> 1025890 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1025891 T1873 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1025892 T1873 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281
   [junit4]   2> 1025892 T1873 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/'
   [junit4]   2> 1025906 T1873 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/solr.xml
   [junit4]   2> 1025954 T1873 oasc.CoreContainer.<init> New CoreContainer 239406661
   [junit4]   2> 1025955 T1873 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/]
   [junit4]   2> 1025957 T1873 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1025958 T1873 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1025958 T1873 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1025959 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1025960 T1873 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1025961 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1025961 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1025962 T1873 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1025963 T1873 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1025964 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1025971 T1873 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41127/solr
   [junit4]   2> 1025972 T1873 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1025973 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1025976 T1913 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e8c1e8c name:ZooKeeperConnection Watcher:127.0.0.1:41127 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1025976 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1025986 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1025993 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1025995 T1915 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@317b317b name:ZooKeeperConnection Watcher:127.0.0.1:41127/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1025996 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1026009 T1873 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1027021 T1873 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60883_nl_c%2Fzk
   [junit4]   2> 1027024 T1873 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60883_nl_c%2Fzk
   [junit4]   2> 1027031 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1027065 T1902 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1027067 T1915 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1027071 T1895 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1027084 T1916 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1027085 T1916 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1027088 T1916 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1027124 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1027130 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45558/nl_c/zk",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:45558_nl_c%2Fzk",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1027143 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60883/nl_c/zk",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:60883_nl_c%2Fzk",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1027143 T1896 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1027144 T1896 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1027155 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1027155 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1027155 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1028089 T1916 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1028089 T1916 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/collection1
   [junit4]   2> 1028090 T1916 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1028093 T1916 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1028093 T1916 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1028098 T1916 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/collection1/'
   [junit4]   2> 1028100 T1916 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/collection1/lib/README' to classloader
   [junit4]   2> 1028100 T1916 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/collection1/lib/classes/' to classloader
   [junit4]   2> 1028136 T1916 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1028184 T1916 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1028190 T1916 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1028197 T1916 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1028675 T1916 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1028676 T1916 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1028676 T1916 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1028682 T1916 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1028686 T1916 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1028745 T1916 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1028752 T1916 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1028760 T1916 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1028764 T1916 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1028764 T1916 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1028765 T1916 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1028769 T1916 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1028769 T1916 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1028770 T1916 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1028771 T1916 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1376308653281/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty1/
   [junit4]   2> 1028771 T1916 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e475e47
   [junit4]   2> 1028773 T1916 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty1
   [junit4]   2> 1028773 T1916 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty1/index/
   [junit4]   2> 1028774 T1916 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1028775 T1916 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty1/index
   [junit4]   2> 1028788 T1916 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@74bc74bc lockFactory=org.apache.lucene.store.NativeFSLockFactory@f200f2),segFN=segments_1,generation=1}
   [junit4]   2> 1028788 T1916 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1028796 T1916 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1028797 T1916 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1028798 T1916 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1028799 T1916 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1028800 T1916 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1028800 T1916 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1028801 T1916 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1028802 T1916 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1028803 T1916 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1028804 T1916 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1028807 T1916 oass.SolrIndexSearcher.<init> Opening Searcher@e570e57 main
   [junit4]   2> 1028808 T1916 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1028809 T1916 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1028815 T1917 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@e570e57 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 1028819 T1916 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1028819 T1916 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60883/nl_c/zk collection:collection1 shard:shard1
   [junit4]   2> 1028820 T1916 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1028827 T1916 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1028866 T1916 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1028890 T1916 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1028891 T1916 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1028892 T1916 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:60883/nl_c/zk/collection1/
   [junit4]   2> 1028892 T1916 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1028893 T1916 oasc.SyncStrategy.syncToMe http://127.0.0.1:60883/nl_c/zk/collection1/ has no replicas
   [junit4]   2> 1028893 T1916 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:60883/nl_c/zk/collection1/ shard1
   [junit4]   2> 1028894 T1916 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1030194 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1030251 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1030251 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1030251 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1030265 T1916 oasc.ZkController.register We are http://127.0.0.1:60883/nl_c/zk/collection1/ and leader is http://127.0.0.1:60883/nl_c/zk/collection1/
   [junit4]   2> 1030266 T1916 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60883/nl_c/zk
   [junit4]   2> 1030267 T1916 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1030268 T1916 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1030269 T1916 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1030275 T1916 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1030278 T1873 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 1030279 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1030280 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1030512 T1873 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1030514 T1873 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:33314
   [junit4]   2> 1030515 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1030516 T1873 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1030517 T1873 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882
   [junit4]   2> 1030518 T1873 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/'
   [junit4]   2> 1030529 T1873 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/solr.xml
   [junit4]   2> 1030580 T1873 oasc.CoreContainer.<init> New CoreContainer 293081464
   [junit4]   2> 1030580 T1873 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/]
   [junit4]   2> 1030582 T1873 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1030583 T1873 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1030583 T1873 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1030584 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1030585 T1873 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1030586 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1030586 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1030587 T1873 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1030588 T1873 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1030589 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1030596 T1873 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41127/solr
   [junit4]   2> 1030597 T1873 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1030598 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1030601 T1929 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10f910f9 name:ZooKeeperConnection Watcher:127.0.0.1:41127 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1030602 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1030606 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1030613 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1030616 T1931 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@674f674f name:ZooKeeperConnection Watcher:127.0.0.1:41127/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1030617 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1030629 T1873 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1031640 T1873 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33314_nl_c%2Fzk
   [junit4]   2> 1031643 T1873 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:33314_nl_c%2Fzk
   [junit4]   2> 1031648 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1031649 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1031650 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1031655 T1931 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1031658 T1902 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1031665 T1895 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1031667 T1915 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1031675 T1932 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1031675 T1932 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1031765 T1932 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1031782 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1031786 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60883/nl_c/zk",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:60883_nl_c%2Fzk",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1031801 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33314/nl_c/zk",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:33314_nl_c%2Fzk",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1031802 T1896 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1031802 T1896 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1031816 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1031816 T1931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1031817 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1031816 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1032767 T1932 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1032768 T1932 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/collection1
   [junit4]   2> 1032769 T1932 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1032774 T1932 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1032775 T1932 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1032782 T1932 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/collection1/'
   [junit4]   2> 1032785 T1932 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/collection1/lib/README' to classloader
   [junit4]   2> 1032787 T1932 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/collection1/lib/classes/' to classloader
   [junit4]   2> 1032829 T1932 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1032868 T1932 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1032872 T1932 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1032880 T1932 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1033342 T1932 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1033343 T1932 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1033344 T1932 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1033349 T1932 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1033352 T1932 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1033391 T1932 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1033398 T1932 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1033404 T1932 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1033408 T1932 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1033409 T1932 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1033409 T1932 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1033413 T1932 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1033414 T1932 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1033414 T1932 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1033415 T1932 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1376308657882/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty2/
   [junit4]   2> 1033415 T1932 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e475e47
   [junit4]   2> 1033425 T1932 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty2
   [junit4]   2> 1033426 T1932 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty2/index/
   [junit4]   2> 1033427 T1932 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1033428 T1932 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty2/index
   [junit4]   2> 1033433 T1932 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@13761376 lockFactory=org.apache.lucene.store.NativeFSLockFactory@104f104f),segFN=segments_1,generation=1}
   [junit4]   2> 1033434 T1932 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1033444 T1932 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1033445 T1932 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1033446 T1932 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1033447 T1932 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1033448 T1932 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1033449 T1932 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1033449 T1932 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1033451 T1932 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1033452 T1932 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1033454 T1932 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1033457 T1932 oass.SolrIndexSearcher.<init> Opening Searcher@5bf05bf main
   [junit4]   2> 1033459 T1932 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1033460 T1932 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1033467 T1933 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5bf05bf main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 1033472 T1932 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1033473 T1932 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:33314/nl_c/zk collection:collection1 shard:shard2
   [junit4]   2> 1033473 T1932 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1033482 T1932 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1033501 T1932 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1033508 T1932 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1033509 T1932 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1033509 T1932 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:33314/nl_c/zk/collection1/
   [junit4]   2> 1033510 T1932 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1033510 T1932 oasc.SyncStrategy.syncToMe http://127.0.0.1:33314/nl_c/zk/collection1/ has no replicas
   [junit4]   2> 1033511 T1932 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:33314/nl_c/zk/collection1/ shard2
   [junit4]   2> 1033512 T1932 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1034889 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1034911 T1931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1034911 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1034913 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1034913 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1034933 T1932 oasc.ZkController.register We are http://127.0.0.1:33314/nl_c/zk/collection1/ and leader is http://127.0.0.1:33314/nl_c/zk/collection1/
   [junit4]   2> 1034934 T1932 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:33314/nl_c/zk
   [junit4]   2> 1034935 T1932 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1034935 T1932 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1034936 T1932 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1034940 T1932 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1034942 T1873 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 1034943 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1034944 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1035144 T1873 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1035147 T1873 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:58246
   [junit4]   2> 1035148 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1035149 T1873 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1035151 T1873 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546
   [junit4]   2> 1035152 T1873 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/'
   [junit4]   2> 1035165 T1873 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/solr.xml
   [junit4]   2> 1035203 T1873 oasc.CoreContainer.<init> New CoreContainer 1605263278
   [junit4]   2> 1035204 T1873 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/]
   [junit4]   2> 1035206 T1873 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1035207 T1873 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1035207 T1873 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1035208 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1035209 T1873 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1035210 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1035210 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1035211 T1873 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1035212 T1873 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1035213 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1035220 T1873 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41127/solr
   [junit4]   2> 1035221 T1873 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1035233 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1035235 T1945 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19c319c3 name:ZooKeeperConnection Watcher:127.0.0.1:41127 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1035235 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1035240 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1035247 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1035249 T1947 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2e802e8 name:ZooKeeperConnection Watcher:127.0.0.1:41127/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1035250 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1035262 T1873 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1036274 T1873 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:58246_nl_c%2Fzk
   [junit4]   2> 1036277 T1873 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:58246_nl_c%2Fzk
   [junit4]   2> 1036283 T1931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1036283 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1036284 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1036284 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1036289 T1947 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1036297 T1915 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1036300 T1931 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1036303 T1902 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1036305 T1895 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1036313 T1948 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1036313 T1948 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1036342 T1948 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1036438 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1036442 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:33314/nl_c/zk",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node2",
   [junit4]   2> 	  "node_name":"127.0.0.1:33314_nl_c%2Fzk",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1036454 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58246/nl_c/zk",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node3",
   [junit4]   2> 	  "node_name":"127.0.0.1:58246_nl_c%2Fzk",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 1036455 T1896 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1036455 T1896 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1036467 T1931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1036467 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1036468 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1036467 T1947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1036467 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1037343 T1948 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1037344 T1948 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/collection1
   [junit4]   2> 1037344 T1948 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1037347 T1948 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1037348 T1948 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1037354 T1948 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/collection1/'
   [junit4]   2> 1037356 T1948 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/collection1/lib/README' to classloader
   [junit4]   2> 1037357 T1948 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/collection1/lib/classes/' to classloader
   [junit4]   2> 1037419 T1948 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_45
   [junit4]   2> 1037462 T1948 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1037466 T1948 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1037474 T1948 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1037883 T1948 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1037883 T1948 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1037884 T1948 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1037890 T1948 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1037893 T1948 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1037928 T1948 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1037944 T1948 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1037951 T1948 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1037956 T1948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1037956 T1948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1037957 T1948 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1037961 T1948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1037962 T1948 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1037962 T1948 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1037963 T1948 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1376308662546/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty3/
   [junit4]   2> 1037964 T1948 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5e475e47
   [junit4]   2> 1037965 T1948 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty3
   [junit4]   2> 1037966 T1948 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty3/index/
   [junit4]   2> 1037967 T1948 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty3/index' doesn't exist. Creating new index...
   [junit4]   2> 1037967 T1948 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1376308649311/jetty3/index
   [junit4]   2> 1037970 T1948 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirWrapper(org.apache.lucene.store.RAMDirectory@12d212d2 lockFactory=org.apache.lucene.store.NativeFSLockFactory@4ffd4ffd),segFN=segments_1,generation=1}
   [junit4]   2> 1037971 T1948 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1037980 T1948 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1037980 T1948 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1037981 T1948 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1037982 T1948 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1037983 T1948 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1037984 T1948 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1037985 T1948 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1037986 T1948 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1037986 T1948 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1037988 T1948 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1037990 T1948 oass.SolrIndexSearcher.<init> Opening Searcher@ca30ca3 main
   [junit4]   2> 1037992 T1948 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1037992 T1948 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1037999 T1949 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ca30ca3 main{StandardDirectoryReader(segments_1:1)}
   [junit4]   2> 1038003 T1948 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1038003 T1948 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:58246/nl_c/zk collection:collection1 shard:shard1
   [junit4]   2> 1038004 T1948 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=10000&maxConnectionsPerHost=20&connTimeout=15000&socketTimeout=30000&retry=false
   [junit4]   2> 1038021 T1948 oasc.ZkController.register We are http://127.0.0.1:58246/nl_c/zk/collection1/ and leader is http://127.0.0.1:60883/nl_c/zk/collection1/
   [junit4]   2> 1038021 T1948 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:58246/nl_c/zk
   [junit4]   2> 1038022 T1948 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 1038022 T1948 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C3843 name=collection1 org.apache.solr.core.SolrCore@425a425a url=http://127.0.0.1:58246/nl_c/zk/collection1 node=127.0.0.1:58246_nl_c%2Fzk C3843_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:58246/nl_c/zk, state=down, node_name=127.0.0.1:58246_nl_c%2Fzk, core=collection1}
   [junit4]   2> 1038023 T1950 C3843 P58246 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 1038024 T1950 C3843 P58246 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 1038024 T1950 C3843 P58246 oasc.ZkController.publish publishing core=collection1 state=recovering
   [junit4]   2> 1038025 T1950 C3843 P58246 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1038025 T1948 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1038027 T1950 C3843 P58246 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1038039 T1873 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 1038040 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1038042 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=128&maxConnectionsPerHost=32&followRedirects=false
   [junit4]   2> 1038045 T1909 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node3, state: recovering, checkLive: true, onlyIfLeader: true
   [junit4]   2> 1038234 T1873 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1038236 T1873 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:51700
   [junit4]   2> 1038237 T1873 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1038238 T1873 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1038239 T1873 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376308665644
   [junit4]   2> 1038240 T1873 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376308665644/'
   [junit4]   2> 1038252 T1873 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376308665644/solr.xml
   [junit4]   2> 1038300 T1873 oasc.CoreContainer.<init> New CoreContainer 1371820484
   [junit4]   2> 1038301 T1873 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty4-1376308665644/]
   [junit4]   2> 1038302 T1873 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 120000
   [junit4]   2> 1038303 T1873 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1038304 T1873 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1038304 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1038305 T1873 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1038306 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1038307 T1873 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1038307 T1873 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1038308 T1873 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1038309 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnectionsPerHost=20&maxConnections=10000&socketTimeout=120000&connTimeout=15000&retry=false
   [junit4]   2> 1038316 T1873 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41127/solr
   [junit4]   2> 1038317 T1873 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1038318 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038321 T1962 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e121e12 name:ZooKeeperConnection Watcher:127.0.0.1:41127 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038322 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038331 T1873 oascsi.HttpClientUtil.createClient Creating new http client, config:maxConnections=500&maxConnectionsPerHost=16&socketTimeout=120000&connTimeout=15000
   [junit4]   2> 1038338 T1873 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1038342 T1964 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8320832 name:ZooKeeperConnection Watcher:127.0.0.1:41127/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1038344 T1873 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1038372 T1873 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1039394 T1873 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:51700_nl_c%2Fzk
   [junit4]   2> 1039398 T1873 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:51700_nl_c%2Fzk
   [junit4]   2> 1039448 T1931 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1039448 T1902 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1039449 T1915 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1039450 T1895 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1039449 T1947 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1039464 T1964 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1039466 T1895 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1039470 T1902 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1039472 T1931 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1039473 T1915 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1039479 T1947 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 1039486 T1965 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1039487 T1965 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1039490 T1965 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1039506 T1896 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1039511 T1896 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:58246/nl_c/zk",
   [junit4]   2> 	  "state":"recoverin

[...truncated too long message...]

odecollection/leaders (1)
   [junit4]   1>      /solr/collections/onenodecollection/leaders/shard1 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "base_url":"http://127.0.0.1:60883/nl_c/zk",
   [junit4]   1>            "node_name":"127.0.0.1:60883_nl_c%2Fzk",
   [junit4]   1>            "core":"onenodecollectioncore"}
   [junit4]   1>   /solr/aliases.json (0)
   [junit4]   1>   /solr/live_nodes (5)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:33314_nl_c%2Fzk (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:45558_nl_c%2Fzk (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:58246_nl_c%2Fzk (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:60883_nl_c%2Fzk (0)
   [junit4]   1>    /solr/live_nodes/127.0.0.1:51700_nl_c%2Fzk (0)
   [junit4]   1>   /solr/clusterstate.json (0)
   [junit4]   1>   DATA:
   [junit4]   1>       {
   [junit4]   1>         "control_collection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:45558/nl_c/zk",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:45558_nl_c%2Fzk",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "onenodecollection":{
   [junit4]   1>           "shards":{"shard1":{
   [junit4]   1>               "range":"80000000-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{"core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:60883/nl_c/zk",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:60883_nl_c%2Fzk",
   [junit4]   1>                   "roles":"none",
   [junit4]   1>                   "core":"onenodecollectioncore",
   [junit4]   1>                   "leader":"true"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"},
   [junit4]   1>         "collection1":{
   [junit4]   1>           "shards":{
   [junit4]   1>             "shard1":{
   [junit4]   1>               "range":"80000000-ffffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node1":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:60883/nl_c/zk",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:60883_nl_c%2Fzk",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node3":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:58246/nl_c/zk",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:58246_nl_c%2Fzk",
   [junit4]   1>                   "core":"collection1"}}},
   [junit4]   1>             "shard2":{
   [junit4]   1>               "range":"0-7fffffff",
   [junit4]   1>               "state":"active",
   [junit4]   1>               "replicas":{
   [junit4]   1>                 "core_node2":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:33314/nl_c/zk",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:33314_nl_c%2Fzk",
   [junit4]   1>                   "core":"collection1",
   [junit4]   1>                   "leader":"true"},
   [junit4]   1>                 "core_node4":{
   [junit4]   1>                   "base_url":"http://127.0.0.1:51700/nl_c/zk",
   [junit4]   1>                   "state":"active",
   [junit4]   1>                   "node_name":"127.0.0.1:51700_nl_c%2Fzk",
   [junit4]   1>                   "core":"collection1"}}}},
   [junit4]   1>           "maxShardsPerNode":"1",
   [junit4]   1>           "router":"compositeId",
   [junit4]   1>           "replicationFactor":"1"}}
   [junit4]   1>  /zookeeper (1)
   [junit4]   1>  DATA:
   [junit4]   1>      
   [junit4]   1> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=BasicDistributedZk2Test -Dtests.method=testDistribSearch -Dtests.seed=762003F01ED3573D -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=et_EE -Dtests.timezone=PRT -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   32.8s J0 | BasicDistributedZk2Test.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Server at http://127.0.0.1:33314/nl_c/zk/onenodecollectioncore returned non ok status:404, message:Can not find: /nl_c/zk/onenodecollectioncore/update
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([762003F01ED3573D:F7C68DE8698C3701]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:385)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:180)
   [junit4]    > 	at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:117)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:116)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:102)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.testNodeWithoutCollectionForwarding(BasicDistributedZk2Test.java:196)
   [junit4]    > 	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:738)
   [junit4]   2> 1054550 T1873 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 32884 T1872 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 1055365 T1964 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1055366 T1964 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene42: {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=et_EE, timezone=PRT
   [junit4]   2> NOTE: Linux 3.2.0-51-generic x86/IBM Corporation 1.6.0 (32-bit)/cpus=8,threads=2,free=23301312,total=59673600
   [junit4]   2> NOTE: All tests run in this JVM: [QueryElevationComponentTest, TestCopyFieldCollectionResource, SynonymTokenizerTest, TestSurroundQueryParser, CopyFieldTest, TestRemoteStreaming, ResponseLogComponentTest, ChaosMonkeyNothingIsSafeTest, HdfsDirectoryTest, SuggesterTest, DirectUpdateHandlerOptimizeTest, TestJoin, StatelessScriptUpdateProcessorFactoryTest, TestTrie, SuggesterFSTTest, TestSystemIdResolver, HdfsUnloadDistributedZkTest, SolrInfoMBeanTest, AnalysisAfterCoreReloadTest, ZkControllerTest, TestFastLRUCache, CSVRequestHandlerTest, BasicFunctionalityTest, SolrCoreTest, BasicDistributedZkTest, BadCopyFieldTest, TestLMDirichletSimilarityFactory, BlockDirectoryTest, TestSolrXmlPersistence, LeaderElectionTest, TestRTGBase, ExternalFileFieldSortTest, TestCollationKeyRangeQueries, TestFunctionQuery, TestStressVersions, TestBM25SimilarityFactory, TestQueryUtils, TestSolrQueryParser, OverseerCollectionProcessorTest, QueryParsingTest, UpdateParamsTest, DOMUtilTest, TestReversedWildcardFilterFactory, TestDocSet, TestSchemaVersionResource, TestDynamicFieldResource, TimeZoneUtilsTest, TestSolrXMLSerializer, HdfsLockFactoryTest, ClusterStateUpdateTest, SimpleFacetsTest, HdfsBasicDistributedZkTest, PrimitiveFieldTypeTest, HdfsCollectionsAPIDistributedZkTest, SpatialFilterTest, TestValueSourceCache, TestSolrQueryParserDefaultOperatorResource, PolyFieldTest, AutoCommitTest, TestClassNameShortening, FileUtilsTest, ShardSplitTest, DistributedSpellCheckComponentTest, TermVectorComponentTest, BadComponentTest, TestSolrJ, TestCharFilters, TestRealTimeGet, JsonLoaderTest, TestWriterPerf, MBeansHandlerTest, TestSolrQueryParserResource, DocumentBuilderTest, TestBadConfig, FieldMutatingUpdateProcessorTest, ParsingFieldUpdateProcessorsTest, SoftAutoCommitTest, BasicDistributedZk2Test]
   [junit4] Completed on J0 in 33.91s, 1 test, 1 error <<< FAILURES!

[...truncated 502 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:395: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:388: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:39: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:181: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:449: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1233: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:876: There were test failures: 319 suites, 1350 tests, 1 error, 35 ignored (6 assumptions)

Total time: 59 minutes 8 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/ibm-j9-jdk6 -Xjit:exclude={org/apache/lucene/util/fst/FST.pack(IIF)Lorg/apache/lucene/util/fst/FST;}
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure