You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2014/02/16 01:02:37 UTC

[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1901 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1901/

2 tests failed.
REGRESSION:  org.apache.solr.TestDistributedSearch.testDistribSearch

Error Message:
Expected the partialResults header to be null

Stack Trace:
java.lang.AssertionError: Expected the partialResults header to be null
	at __randomizedtesting.SeedInfo.seed([B50ABD9EBC7AD7AD:34EC3386CB25B791]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.TestDistributedSearch.validateControlData(TestDistributedSearch.java:525)
	at org.apache.solr.TestDistributedSearch.queryPartialResults(TestDistributedSearch.java:445)
	at org.apache.solr.TestDistributedSearch.doTest(TestDistributedSearch.java:379)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:876)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:61657/collection1, https://127.0.0.1:19326/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:61657/collection1, https://127.0.0.1:19326/collection1]
	at __randomizedtesting.SeedInfo.seed([B50ABD9EBC7AD7AD:34EC3386CB25B791]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:362)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:635)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1356)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
	at org.apache.solr.cloud.BasicDistributedZk2Test.brindDownShardIndexSomeDocsAndRecover(BasicDistributedZk2Test.java:290)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:107)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: no servers hosting shard: 
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:495)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:293)
	... 50 more




Build Log:
[...truncated 9934 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZk2Test
   [junit4]   2> 18831 T17 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 18833 T17 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 18847 T17 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZk2Test-1392506220718
   [junit4]   2> 18860 T17 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 18871 T18 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 19173 T17 oasc.ZkTestServer.run start zk server on port:51058
   [junit4]   2> 19467 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 19521 T19 oazs.NIOServerCnxn.doIO WARN Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 21110 T23 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2954b860 name:ZooKeeperConnection Watcher:127.0.0.1:51058 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 21111 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 21121 T17 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 21286 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 21289 T26 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@198c8799 name:ZooKeeperConnection Watcher:127.0.0.1:51058/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 21290 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 21324 T17 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 21332 T17 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 21338 T17 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 21344 T17 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 21351 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 21390 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 21407 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 21408 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 21516 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 21517 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 21524 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 21525 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 21531 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 21532 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 21539 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 21540 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 21546 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 21547 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 21554 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 21555 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 21562 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 21563 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 21569 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 21570 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 21576 T17 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 21577 T17 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 21605 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 21608 T28 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3599e94c name:ZooKeeperConnection Watcher:127.0.0.1:51058/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 21609 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 22840 T17 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 23242 T17 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 23276 T17 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:25633
   [junit4]   2> 23446 T17 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 23447 T17 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 23448 T17 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488
   [junit4]   2> 23449 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/'
   [junit4]   2> 23482 T17 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/solr.xml
   [junit4]   2> 23581 T17 oasc.CoreContainer.<init> New CoreContainer 634485636
   [junit4]   2> 23581 T17 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/]
   [junit4]   2> 23646 T17 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 23647 T17 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 23647 T17 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 23648 T17 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 23648 T17 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 23649 T17 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 23649 T17 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 23650 T17 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 23650 T17 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 23667 T17 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 23668 T17 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 23668 T17 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 23669 T17 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51058/solr
   [junit4]   2> 23707 T17 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 23708 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 23713 T40 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6b06eae3 name:ZooKeeperConnection Watcher:127.0.0.1:51058 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 23713 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 23741 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 23743 T42 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b898034 name:ZooKeeperConnection Watcher:127.0.0.1:51058/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 23744 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 23759 T17 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 23773 T17 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 23801 T17 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 23804 T17 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25633_
   [junit4]   2> 23812 T17 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25633_
   [junit4]   2> 23817 T17 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 23822 T17 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 23846 T17 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25633_
   [junit4]   2> 23847 T17 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 23852 T17 oasc.Overseer.start Overseer (id=91259287710269444-127.0.0.1:25633_-n_0000000000) starting
   [junit4]   2> 23881 T17 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 23901 T44 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 23902 T17 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 23906 T17 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 23908 T44 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 23909 T17 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 23936 T43 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 23960 T45 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 23961 T45 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 23968 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 23969 T45 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 23978 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 23979 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:25633",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25633_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 23980 T43 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 24013 T43 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 24021 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 24035 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 24971 T45 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 24972 T45 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/collection1
   [junit4]   2> 24972 T45 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 24974 T45 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 24975 T45 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 24977 T45 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 24978 T45 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/collection1/'
   [junit4]   2> 24980 T45 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/collection1/lib/classes/' to classloader
   [junit4]   2> 24981 T45 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/collection1/lib/README' to classloader
   [junit4]   2> 25182 T45 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 25291 T45 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 25394 T45 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 25504 T45 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 26308 T45 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 26309 T45 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 26311 T45 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 26327 T45 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 26330 T45 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 26363 T45 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 26369 T45 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 26374 T45 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 26376 T45 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 26377 T45 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 26377 T45 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 26379 T45 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 26379 T45 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 26380 T45 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 26380 T45 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392506223488/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/control/data/
   [junit4]   2> 26381 T45 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@56c3560b
   [junit4]   2> 26387 T45 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/control/data
   [junit4]   2> 26388 T45 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/control/data/index/
   [junit4]   2> 26388 T45 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 26389 T45 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/control/data/index
   [junit4]   2> 26389 T45 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 26391 T45 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3ee98e03 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@335c592),segFN=segments_1,generation=1}
   [junit4]   2> 26392 T45 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 26398 T45 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 26452 T45 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 26453 T45 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 26453 T45 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 26454 T45 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 26454 T45 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 26455 T45 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 26455 T45 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 26456 T45 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 26462 T45 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 26464 T45 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 26464 T45 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 26464 T45 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 26466 T45 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 26466 T45 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 26468 T45 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 26483 T45 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26490 T45 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 26491 T45 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 26494 T45 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=517500355, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 26495 T45 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3ee98e03 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@335c592),segFN=segments_1,generation=1}
   [junit4]   2> 26496 T45 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 26496 T45 oass.SolrIndexSearcher.<init> Opening Searcher@6072205a[collection1] main
   [junit4]   2> 26506 T46 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6072205a[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 26513 T45 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 26514 T17 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 26514 T49 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:25633 collection:control_collection shard:shard1
   [junit4]   2> 26515 T17 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 26548 T49 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 26579 T49 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 26583 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26584 T49 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 26584 T49 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 26586 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@615e8f2b url=https://127.0.0.1:25633/collection1 node=127.0.0.1:25633_ C0_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:25633, core=collection1, node_name=127.0.0.1:25633_}
   [junit4]   2> 26590 T49 C0 P25633 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:25633/collection1/
   [junit4]   2> 26594 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26594 T49 C0 P25633 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 26595 T49 C0 P25633 oasc.SyncStrategy.syncToMe https://127.0.0.1:25633/collection1/ has no replicas
   [junit4]   2> 26595 T49 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:25633/collection1/ shard1
   [junit4]   2> 26596 T49 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 26602 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 26607 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 26616 T51 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@268f10d0 name:ZooKeeperConnection Watcher:127.0.0.1:51058/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 26617 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 26619 T17 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 26626 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26629 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 26631 T17 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 26640 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26744 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 26744 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 26780 T49 oasc.ZkController.register We are https://127.0.0.1:25633/collection1/ and leader is https://127.0.0.1:25633/collection1/
   [junit4]   2> 26780 T49 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:25633
   [junit4]   2> 26781 T49 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 26781 T49 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 26782 T49 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 26798 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26798 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26799 T49 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 26799 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26801 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 26803 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:25633",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:25633_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 26807 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 26911 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 26911 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 26918 T17 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 26928 T17 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 26939 T17 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 26947 T17 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:40327
   [junit4]   2> 26953 T17 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 26954 T17 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 26955 T17 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501
   [junit4]   2> 26955 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/'
   [junit4]   2> 27004 T17 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/solr.xml
   [junit4]   2> 27150 T17 oasc.CoreContainer.<init> New CoreContainer 1467880681
   [junit4]   2> 27151 T17 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/]
   [junit4]   2> 27153 T17 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 27154 T17 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 27155 T17 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 27155 T17 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 27156 T17 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 27157 T17 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 27157 T17 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 27158 T17 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 27159 T17 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 27181 T17 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 27182 T17 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 27183 T17 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 27183 T17 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51058/solr
   [junit4]   2> 27184 T17 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 27186 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 27189 T62 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@222f2421 name:ZooKeeperConnection Watcher:127.0.0.1:51058 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 27190 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 27193 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 27201 T64 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44a0abc2 name:ZooKeeperConnection Watcher:127.0.0.1:51058/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 27201 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 27210 T17 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 28217 T17 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40327_
   [junit4]   2> 28245 T17 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40327_
   [junit4]   2> 28250 T51 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 28250 T64 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 28250 T42 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 28269 T65 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 28269 T65 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 28272 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 28272 T65 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 28272 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 28273 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 28274 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 28276 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:40327",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40327_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 28277 T43 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 28278 T43 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 28282 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 28387 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 28387 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 28387 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 29274 T65 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 29274 T65 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/collection1
   [junit4]   2> 29275 T65 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 29276 T65 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 29277 T65 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 29279 T65 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 29279 T65 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/collection1/'
   [junit4]   2> 29282 T65 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/collection1/lib/README' to classloader
   [junit4]   2> 29283 T65 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/collection1/lib/classes/' to classloader
   [junit4]   2> 29380 T65 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 29482 T65 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 29585 T65 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 29676 T65 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 30451 T65 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 30451 T65 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 30453 T65 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 30469 T65 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 30473 T65 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 30507 T65 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 30513 T65 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 30519 T65 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 30521 T65 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 30521 T65 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 30522 T65 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 30524 T65 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 30524 T65 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 30524 T65 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 30525 T65 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392506228501/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty1/
   [junit4]   2> 30525 T65 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@56c3560b
   [junit4]   2> 30527 T65 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty1
   [junit4]   2> 30527 T65 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty1/index/
   [junit4]   2> 30527 T65 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 30528 T65 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty1/index
   [junit4]   2> 30528 T65 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 30530 T65 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f998c67 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@412ca405),segFN=segments_1,generation=1}
   [junit4]   2> 30530 T65 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 30538 T65 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 30539 T65 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 30539 T65 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 30539 T65 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 30540 T65 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 30540 T65 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 30541 T65 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 30542 T65 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 30543 T65 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 30545 T65 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 30547 T65 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 30547 T65 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 30548 T65 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 30550 T65 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 30550 T65 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 30552 T65 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 30563 T65 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30568 T65 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 30568 T65 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 30570 T65 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=517500355, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 30571 T65 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3f998c67 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@412ca405),segFN=segments_1,generation=1}
   [junit4]   2> 30572 T65 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 30572 T65 oass.SolrIndexSearcher.<init> Opening Searcher@57ce49a7[collection1] main
   [junit4]   2> 30587 T66 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@57ce49a7[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 30599 T65 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 30600 T17 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 30600 T17 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 30600 T69 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:40327 collection:collection1 shard:shard1
   [junit4]   2> 30603 T69 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 30719 T69 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 30756 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30757 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30757 T69 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 30757 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30758 T69 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 30759 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@9674b2d url=https://127.0.0.1:40327/collection1 node=127.0.0.1:40327_ C1_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:40327, core=collection1, node_name=127.0.0.1:40327_}
   [junit4]   2> 30759 T69 C1 P40327 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:40327/collection1/
   [junit4]   2> 30760 T69 C1 P40327 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 30761 T69 C1 P40327 oasc.SyncStrategy.syncToMe https://127.0.0.1:40327/collection1/ has no replicas
   [junit4]   2> 30761 T69 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:40327/collection1/ shard1
   [junit4]   2> 30762 T69 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 30765 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30776 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 30776 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 30776 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 30789 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 30794 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30807 T17 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 30808 T17 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 30815 T17 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 30819 T17 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:19326
   [junit4]   2> 30822 T17 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 30823 T17 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 30823 T17 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473
   [junit4]   2> 30824 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/'
   [junit4]   2> 30852 T17 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/solr.xml
   [junit4]   2> 30898 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 30898 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 30898 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 30942 T69 oasc.ZkController.register We are https://127.0.0.1:40327/collection1/ and leader is https://127.0.0.1:40327/collection1/
   [junit4]   2> 30942 T69 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:40327
   [junit4]   2> 30943 T69 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 30943 T69 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 30944 T69 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 30947 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30947 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30947 T69 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 30948 T17 oasc.CoreContainer.<init> New CoreContainer 1863151102
   [junit4]   2> 30947 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30949 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 30949 T17 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/]
   [junit4]   2> 30950 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:40327",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40327_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 30951 T17 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 30952 T17 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 30952 T17 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 30953 T17 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 30953 T17 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 30954 T17 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 30954 T17 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 30954 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 30955 T17 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 30956 T17 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 30971 T17 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 30971 T17 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 30972 T17 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 30972 T17 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51058/solr
   [junit4]   2> 30973 T17 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 30974 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 30977 T80 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25b2dd9e name:ZooKeeperConnection Watcher:127.0.0.1:51058 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 30977 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 30980 T17 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 30982 T82 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18716ec7 name:ZooKeeperConnection Watcher:127.0.0.1:51058/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 30983 T17 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 30996 T17 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 31061 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 31061 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 31061 T82 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 31061 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 32001 T17 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19326_
   [junit4]   2> 32003 T17 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19326_
   [junit4]   2> 32008 T64 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 32008 T82 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 32008 T42 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 32008 T51 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 32029 T83 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 32029 T83 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 32032 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 32032 T83 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 32032 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 32033 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 32035 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 32037 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:19326",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19326_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 32038 T43 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 32039 T43 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 32044 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 32149 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 32149 T82 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 32149 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 32149 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 33034 T83 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 33034 T83 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/collection1
   [junit4]   2> 33035 T83 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 33037 T83 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 33037 T83 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 33039 T83 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 33040 T83 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/collection1/'
   [junit4]   2> 33042 T83 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/collection1/lib/README' to classloader
   [junit4]   2> 33043 T83 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/collection1/lib/classes/' to classloader
   [junit4]   2> 33131 T83 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 33221 T83 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 33324 T83 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 33402 T83 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 34340 T83 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 34341 T83 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 34344 T83 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 34370 T83 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 34376 T83 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 34419 T83 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34428 T83 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34438 T83 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34440 T83 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34441 T83 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34441 T83 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34443 T83 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34444 T83 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34444 T83 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 34445 T83 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392506232473/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty2/
   [junit4]   2> 34445 T83 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@56c3560b
   [junit4]   2> 34448 T83 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty2
   [junit4]   2> 34449 T83 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty2/index/
   [junit4]   2> 34449 T83 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 34450 T83 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392506220717/jetty2/index
   [junit4]   2> 34450 T83 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=22, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 34452 T83 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1068ace6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7b6bc9a7),segFN=segments_1,generation=1}
   [junit4]   2> 34453 T83 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34465 T83 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 34466 T83 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 34466 T83 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 34467 T83 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34467 T83 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34468 T83 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 34469 T83 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 34470 T83 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 34471 T83 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 34473 T83 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 34476 T83 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 34476 T83 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 34477 T83 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 34480 T83 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 34481 T83 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 34483 T83 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 34506 T83 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 34513 T83 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 34514 T83 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 34517 T83 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=517500355, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 34518 T83 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1068ace6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7b6bc9a7),segFN=segments_1,generation=1}
   [junit4]   2> 34519 T83 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34519 T83 oass.SolrIndexSearcher.<init> Opening Searcher@4c8723cf[collection1] main
   [junit4]   2> 34537 T84 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4c8723cf[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 34547 T83 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 34548 T17 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 34549 T17 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 34548 T87 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:19326 collection:collection1 shard:shard2
   [junit4]   2> 34552 T87 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 34571 T87 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 34581 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34582 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34582 T87 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 34582 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34583 T87 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2 name=collection1 org.apache.solr.core.SolrCore@7c8e1724 url=https://127.0.0.1:19326/collection1 node=127.0.0.1:19326_ C2_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:19326, core=collection1, node_name=127.0.0.1:19326_}
   [junit4]   2> 34584 T87 C2 P19326 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:19326/collection1/
   [junit4]   2> 34585 T87 C2 P19326 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 34585 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 34585 T87 C2 P19326 oasc.SyncStrategy.syncToMe https://127.0.0.1:19326/collection1/ has no replicas
   [junit4]   2> 34586 T87 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:19326/collection1/ shard2
   [junit4]   2> 34586 T87 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 34592 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34601 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34601 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34601 T82 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34601 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34621 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 34627 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34737 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34737 T82 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34737 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34737 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34771 T87 oasc.ZkController.register We are https://127.0.0.1:19326/collection1/ and leader is https://127.0.0.1:19326/collection1/
   [junit4]   2> 34771 T87 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:19326
   [junit4]   2> 34772 T87 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 34772 T87 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 34773 T87 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 34780 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34781 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34781 T87 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 34781 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34784 T43 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 34786 T43 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:19326",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:19326_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 34803 T42 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 34876 T17 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 34877 T17 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 34889 T17 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 34894 T17 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:20686
   [junit4]   2> 34899 T17 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 34899 T17 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 34900 T17 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1392506236423
   [junit4]   2> 34901 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1392506236423/'
   [junit4]   2> 34908 T42 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34908 T51 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34908 T64 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34908 T82 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 34942 T17 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty3-1392506236423/solr.xml
   [junit4]   2> 35049 T17 oasc.CoreContainer.<init> New Co

[...truncated too long message...]

actory.close Closing directory: ./org.apache.solr.TestDistributedSearch-1392508422241/control/data/index
   [junit4]   2> 2276062 T6413 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.TestDistributedSearch-1392508422241/control/data [CachedDir<<refCount=0;path=./org.apache.solr.TestDistributedSearch-1392508422241/control/data;done=false>>]
   [junit4]   2> 2276062 T6413 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.TestDistributedSearch-1392508422241/control/data
   [junit4]   2> 2276063 T6413 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2276117 T6413 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=2035457606
   [junit4]   2> 2276118 T6413 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@63955344
   [junit4]   2> 2276131 T6413 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=3,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=24,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=6125,transaction_logs_total_number=3}
   [junit4]   2> 2276132 T6413 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2276133 T6413 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2276133 T6413 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2276135 T6413 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2276135 T6413 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2276136 T6413 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.TestDistributedSearch-1392508422241/shard0/data [CachedDir<<refCount=0;path=./org.apache.solr.TestDistributedSearch-1392508422241/shard0/data;done=false>>]
   [junit4]   2> 2276136 T6413 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.TestDistributedSearch-1392508422241/shard0/data
   [junit4]   2> 2276137 T6413 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.TestDistributedSearch-1392508422241/shard0/data/index [CachedDir<<refCount=0;path=./org.apache.solr.TestDistributedSearch-1392508422241/shard0/data/index;done=false>>]
   [junit4]   2> 2276137 T6413 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.TestDistributedSearch-1392508422241/shard0/data/index
   [junit4]   2> 2276138 T6413 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2276192 T6413 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=499587660
   [junit4]   2> 2276193 T6413 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@520563ba
   [junit4]   2> 2276213 T6413 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=3,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=23,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=5687,transaction_logs_total_number=3}
   [junit4]   2> 2276215 T6413 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2276215 T6413 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2276215 T6413 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2276217 T6413 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2276218 T6413 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2276218 T6413 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.TestDistributedSearch-1392508422241/shard2/data/index [CachedDir<<refCount=0;path=./org.apache.solr.TestDistributedSearch-1392508422241/shard2/data/index;done=false>>]
   [junit4]   2> 2276219 T6413 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.TestDistributedSearch-1392508422241/shard2/data/index
   [junit4]   2> 2276219 T6413 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.TestDistributedSearch-1392508422241/shard2/data [CachedDir<<refCount=0;path=./org.apache.solr.TestDistributedSearch-1392508422241/shard2/data;done=false>>]
   [junit4]   2> 2276220 T6413 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.TestDistributedSearch-1392508422241/shard2/data
   [junit4]   2> 2276221 T6413 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2276311 T6413 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDistributedSearch -Dtests.method=testDistribSearch -Dtests.seed=B50ABD9EBC7AD7AD -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=no_NO -Dtests.timezone=America/Montreal -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 56.0s J0 | TestDistributedSearch.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Expected the partialResults header to be null
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B50ABD9EBC7AD7AD:34EC3386CB25B791]:0)
   [junit4]    > 	at org.apache.solr.TestDistributedSearch.validateControlData(TestDistributedSearch.java:525)
   [junit4]    > 	at org.apache.solr.TestDistributedSearch.queryPartialResults(TestDistributedSearch.java:445)
   [junit4]    > 	at org.apache.solr.TestDistributedSearch.doTest(TestDistributedSearch.java:379)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:876)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 2276376 T6413 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene45, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=no_NO, timezone=America/Montreal
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=2,free=208621056,total=279969792
   [junit4]   2> NOTE: All tests run in this JVM: [SolrRequestParserTest, BasicDistributedZk2Test, ZkCLITest, TestUpdate, CoreMergeIndexesAdminHandlerTest, TestBinaryField, SpatialFilterTest, SliceStateUpdateTest, LeaderElectionIntegrationTest, TestShardHandlerFactory, TestPhraseSuggestions, PolyFieldTest, TestStressVersions, TestRangeQuery, TestCodecSupport, UniqFieldsUpdateProcessorFactoryTest, TestSchemaNameResource, LegacyHTMLStripCharFilterTest, TestUniqueKeyFieldResource, TestAnalyzeInfixSuggestions, ClusterStateTest, TestStressReorder, SolrIndexSplitterTest, WordBreakSolrSpellCheckerTest, TestFoldingMultitermQuery, DistanceFunctionTest, ResourceLoaderTest, SimpleFacetsTest, PrimitiveFieldTypeTest, TestQuerySenderNoQuery, CacheHeaderTest, TestSolrIndexConfig, AliasIntegrationTest, SolrCoreCheckLockOnStartupTest, TestWriterPerf, TestRandomMergePolicy, PeerSyncTest, TermVectorComponentDistributedTest, TestAnalyzedSuggestions, TestCSVResponseWriter, CollectionsAPIDistributedZkTest, BJQParserTest, TestSolrXml, SuggesterTSTTest, XsltUpdateRequestHandlerTest, UpdateRequestProcessorFactoryTest, TestFastLRUCache, TestFuzzyAnalyzedSuggestions, TestLRUCache, OverseerCollectionProcessorTest, SchemaVersionSpecificBehaviorTest, PrimUtilsTest, TestSearchPerf, CursorMarkTest, PreAnalyzedFieldTest, FieldMutatingUpdateProcessorTest, UUIDFieldTest, TestFastOutputStream, SynonymTokenizerTest, TestXIncludeConfig, TestFiltering, TestStressRecovery, HdfsLockFactoryTest, TestFieldResource, FileUtilsTest, TestManagedSchemaFieldResource, SignatureUpdateProcessorFactoryTest, MultiTermTest, DebugComponentTest, TestSerializedLuceneMatchVersion, TestNonDefinedSimilarityFactory, BadCopyFieldTest, MigrateRouteKeyTest, UnloadDistributedZkTest, TermVectorComponentTest, SuggestComponentTest, TestJoin, HighlighterConfigTest, ShardRoutingTest, OverseerRolesTest, TestQueryUtils, ScriptEngineTest, BadIndexSchemaTest, ZkControllerTest, SolrXmlInZkTest, TestImplicitCoreProperties, SyncSliceTest, TestDistributedSearch]
   [junit4] Completed on J0 in 56.05s, 1 test, 1 failure <<< FAILURES!

[...truncated 550 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1308: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:941: There were test failures: 362 suites, 1563 tests, 1 error, 1 failure, 35 ignored (7 assumptions)

Total time: 72 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-Tests-4.x-Java7 - Build # 1902 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-Tests-4.x-Java7/1902/

4 tests failed.
REGRESSION:  org.apache.solr.TestDistributedGrouping.testDistribSearch

Error Message:
Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 Server Error</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /select. Reason: <pre>    Server Error</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 Server Error</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /select. Reason:
<pre>    Server Error</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([5D57ED374AC8547:8433F0CB03F3E57B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:459)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:515)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
	at org.apache.solr.TestDistributedGrouping.doTest(TestDistributedGrouping.java:156)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:876)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:44914/collection1, https://127.0.0.1:35430/collection1, https://127.0.0.1:24431/collection1, https://127.0.0.1:33903/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:44914/collection1, https://127.0.0.1:35430/collection1, https://127.0.0.1:24431/collection1, https://127.0.0.1:33903/collection1]
	at __randomizedtesting.SeedInfo.seed([5D57ED374AC8547:8433F0CB03F3E57B]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:362)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:635)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1356)
	at org.apache.solr.cloud.BasicDistributedZkTest.queryServer(BasicDistributedZkTest.java:1171)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:264)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 Server Error</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/select. Reason:
<pre>    Server Error</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:459)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:293)
	... 49 more


FAILED:  org.apache.solr.TestDistributedSearch.testDistribSearch

Error Message:
Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 Server Error</title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /yr_/g/select. Reason: <pre>    Server Error</pre></p> <hr /><i><small>Powered by Jetty://</small></i>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 Server Error</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /yr_/g/select. Reason:
<pre>    Server Error</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([5D57ED374AC8547:8433F0CB03F3E57B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:459)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:515)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
	at org.apache.solr.TestDistributedSearch.doTest(TestDistributedSearch.java:177)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:876)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


FAILED:  org.apache.solr.cloud.BasicDistributedZk2Test.testDistribSearch

Error Message:
No live SolrServers available to handle this request:[https://127.0.0.1:19863/naom/c/collection1, https://127.0.0.1:11374/naom/c/collection1, https://127.0.0.1:11377/naom/c/collection1, https://127.0.0.1:11380/naom/c/collection1]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[https://127.0.0.1:19863/naom/c/collection1, https://127.0.0.1:11374/naom/c/collection1, https://127.0.0.1:11377/naom/c/collection1, https://127.0.0.1:11380/naom/c/collection1]
	at __randomizedtesting.SeedInfo.seed([5D57ED374AC8547:8433F0CB03F3E57B]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:362)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:635)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.queryServer(AbstractFullDistribZkTestBase.java:1356)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkQueries(AbstractFullDistribZkTestBase.java:797)
	at org.apache.solr.cloud.BasicDistributedZk2Test.doTest(BasicDistributedZk2Test.java:101)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 Server Error</title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /naom/c/collection1/select. Reason:
<pre>    Server Error</pre></p>
<hr /><i><small>Powered by Jetty://</small></i>
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
                                                
</body>
</html>

	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:459)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:293)
	... 50 more




Build Log:
[...truncated 10086 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZk2Test
   [junit4]   2> 487312 T1809 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 487313 T1809 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /naom/c
   [junit4]   2> 487321 T1809 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./solrtest-BasicDistributedZk2Test-1392572548345
   [junit4]   2> 487322 T1809 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 487323 T1810 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 487424 T1809 oasc.ZkTestServer.run start zk server on port:11366
   [junit4]   2> 487425 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 487431 T1816 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21e49c6b name:ZooKeeperConnection Watcher:127.0.0.1:11366 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 487432 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 487432 T1809 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 487438 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 487478 T1818 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@660ed876 name:ZooKeeperConnection Watcher:127.0.0.1:11366/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 487478 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 487479 T1809 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 487503 T1809 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 487506 T1809 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 487509 T1809 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 487545 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 487546 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 487635 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 487636 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 487744 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 487745 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 487748 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 487749 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 487752 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 487753 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 487765 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 487766 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 487773 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 487774 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 487777 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 487778 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 487781 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 487782 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 487790 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 487791 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 487794 T1809 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 487794 T1809 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 487805 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 487807 T1820 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16f6bcc6 name:ZooKeeperConnection Watcher:127.0.0.1:11366/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 487807 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 488139 T1809 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 488146 T1809 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 488151 T1809 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:11370
   [junit4]   2> 488155 T1809 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 488156 T1809 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 488156 T1809 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840
   [junit4]   2> 488157 T1809 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/'
   [junit4]   2> 488201 T1809 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/solr.xml
   [junit4]   2> 488308 T1809 oasc.CoreContainer.<init> New CoreContainer 1934838671
   [junit4]   2> 488309 T1809 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/]
   [junit4]   2> 488311 T1809 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 488311 T1809 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 488312 T1809 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 488312 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 488313 T1809 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 488313 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 488314 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 488314 T1809 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 488315 T1809 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 488333 T1809 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 488334 T1809 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 488335 T1809 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 488335 T1809 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11366/solr
   [junit4]   2> 488336 T1809 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 488337 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 488340 T1831 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4b1fa2fb name:ZooKeeperConnection Watcher:127.0.0.1:11366 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 488341 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 488355 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 488358 T1833 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@cef2c6b name:ZooKeeperConnection Watcher:127.0.0.1:11366/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 488358 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 488361 T1809 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 488370 T1809 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 488375 T1809 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 488377 T1809 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11370_naom%2Fc
   [junit4]   2> 488385 T1809 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11370_naom%2Fc
   [junit4]   2> 488392 T1809 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 488395 T1809 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 488408 T1809 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:11370_naom%2Fc
   [junit4]   2> 488409 T1809 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 488412 T1809 oasc.Overseer.start Overseer (id=91263634528862212-127.0.0.1:11370_naom%2Fc-n_0000000000) starting
   [junit4]   2> 488424 T1809 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 488435 T1835 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 488435 T1809 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 488436 T1835 oasc.OverseerCollectionProcessor.prioritizeOverseerNodes prioritizing overseer nodes
   [junit4]   2> 488438 T1809 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 488441 T1809 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 488443 T1834 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 488450 T1836 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 488450 T1836 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 488452 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 488452 T1836 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 488453 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 488454 T1834 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:11370/naom/c",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:11370_naom%2Fc",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 488454 T1834 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 488454 T1834 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 488458 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 488466 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 489453 T1836 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 489453 T1836 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/collection1
   [junit4]   2> 489453 T1836 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 489454 T1836 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 489455 T1836 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 489456 T1836 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 489456 T1836 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/collection1/'
   [junit4]   2> 489457 T1836 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/collection1/lib/classes/' to classloader
   [junit4]   2> 489458 T1836 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/collection1/lib/README' to classloader
   [junit4]   2> 489527 T1836 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 489604 T1836 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 489706 T1836 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 489719 T1836 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 490410 T1836 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 490411 T1836 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 490411 T1836 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 490424 T1836 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 490429 T1836 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 490445 T1836 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 490450 T1836 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 490457 T1836 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 490458 T1836 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 490458 T1836 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 490459 T1836 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 490460 T1836 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 490460 T1836 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 490460 T1836 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 490461 T1836 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.BasicDistributedZk2Test-controljetty-1392572548840/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/control/data/
   [junit4]   2> 490461 T1836 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@57202fd
   [junit4]   2> 490462 T1836 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/control/data
   [junit4]   2> 490462 T1836 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/control/data/index/
   [junit4]   2> 490463 T1836 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 490463 T1836 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/control/data/index
   [junit4]   2> 490464 T1836 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 490465 T1836 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2da46e96 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a6556cf),segFN=segments_1,generation=1}
   [junit4]   2> 490465 T1836 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 490470 T1836 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 490470 T1836 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 490471 T1836 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 490471 T1836 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 490471 T1836 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 490472 T1836 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 490472 T1836 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 490473 T1836 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 490473 T1836 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 490474 T1836 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 490474 T1836 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 490475 T1836 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 490475 T1836 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 490476 T1836 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 490476 T1836 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 490477 T1836 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 490495 T1836 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 490501 T1836 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 490502 T1836 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 490503 T1836 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=66.173828125, floorSegmentMB=0.6552734375, forceMergeDeletesPctAllowed=19.629843639066507, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14090802625859133
   [junit4]   2> 490504 T1836 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2da46e96 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1a6556cf),segFN=segments_1,generation=1}
   [junit4]   2> 490504 T1836 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 490504 T1836 oass.SolrIndexSearcher.<init> Opening Searcher@6fbd57ac[collection1] main
   [junit4]   2> 490509 T1837 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6fbd57ac[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 490511 T1836 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 490512 T1809 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 490513 T1809 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 490512 T1840 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:11370/naom/c collection:control_collection shard:shard1
   [junit4]   2> 490514 T1840 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 490519 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 490532 T1842 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d6c0717 name:ZooKeeperConnection Watcher:127.0.0.1:11366/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 490533 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 490534 T1809 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 490535 T1840 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 490537 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 490537 T1809 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 490538 T1840 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 490538 T1840 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C259 name=collection1 org.apache.solr.core.SolrCore@14d53eeb url=https://127.0.0.1:11370/naom/c/collection1 node=127.0.0.1:11370_naom%2Fc C259_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:11370/naom/c, core=collection1, node_name=127.0.0.1:11370_naom%2Fc}
   [junit4]   2> 490539 T1840 C259 P11370 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:11370/naom/c/collection1/
   [junit4]   2> 490539 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 490539 T1840 C259 P11370 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 490540 T1840 C259 P11370 oasc.SyncStrategy.syncToMe https://127.0.0.1:11370/naom/c/collection1/ has no replicas
   [junit4]   2> 490540 T1840 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:11370/naom/c/collection1/ shard1
   [junit4]   2> 490540 T1840 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 490543 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 490544 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 490544 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 490549 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 490553 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 490656 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 490657 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 490702 T1840 oasc.ZkController.register We are https://127.0.0.1:11370/naom/c/collection1/ and leader is https://127.0.0.1:11370/naom/c/collection1/
   [junit4]   2> 490702 T1840 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:11370/naom/c
   [junit4]   2> 490702 T1840 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 490703 T1840 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 490703 T1840 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 490705 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 490705 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 490705 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 490706 T1840 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 490707 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 490708 T1834 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:11370/naom/c",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:11370_naom%2Fc",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 490711 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 490814 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 490814 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 490861 T1809 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 490862 T1809 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 490869 T1809 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 490872 T1809 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:11374
   [junit4]   2> 490876 T1809 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 490876 T1809 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 490877 T1809 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561
   [junit4]   2> 490877 T1809 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/'
   [junit4]   2> 490922 T1809 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/solr.xml
   [junit4]   2> 491025 T1809 oasc.CoreContainer.<init> New CoreContainer 2068740631
   [junit4]   2> 491026 T1809 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/]
   [junit4]   2> 491028 T1809 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 491028 T1809 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 491029 T1809 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 491029 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 491030 T1809 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 491030 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 491031 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 491031 T1809 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 491032 T1809 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 491049 T1809 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 491050 T1809 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 491050 T1809 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 491051 T1809 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11366/solr
   [junit4]   2> 491052 T1809 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 491053 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 491081 T1853 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7339065c name:ZooKeeperConnection Watcher:127.0.0.1:11366 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 491082 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 491086 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 491092 T1855 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27d5e961 name:ZooKeeperConnection Watcher:127.0.0.1:11366/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 491093 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 491098 T1809 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 492102 T1809 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11374_naom%2Fc
   [junit4]   2> 492104 T1809 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11374_naom%2Fc
   [junit4]   2> 492108 T1833 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 492108 T1842 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 492108 T1855 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 492124 T1856 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 492125 T1856 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 492127 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 492127 T1856 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 492127 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 492128 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 492128 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 492129 T1834 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:11374/naom/c",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:11374_naom%2Fc",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 492129 T1834 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 492129 T1834 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 492133 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 492275 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492275 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492275 T1855 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 493128 T1856 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 493129 T1856 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/collection1
   [junit4]   2> 493129 T1856 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 493130 T1856 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 493131 T1856 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 493154 T1856 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 493155 T1856 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/collection1/'
   [junit4]   2> 493157 T1856 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/collection1/lib/README' to classloader
   [junit4]   2> 493158 T1856 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/collection1/lib/classes/' to classloader
   [junit4]   2> 493229 T1856 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_47
   [junit4]   2> 493303 T1856 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 493405 T1856 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 493417 T1856 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 494044 T1856 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 494045 T1856 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 494045 T1856 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 494054 T1856 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 494057 T1856 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 494070 T1856 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 494074 T1856 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 494078 T1856 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 494079 T1856 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 494079 T1856 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 494079 T1856 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 494080 T1856 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 494080 T1856 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 494081 T1856 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 494081 T1856 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty1-1392572551561/collection1/, dataDir=./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/jetty1/
   [junit4]   2> 494081 T1856 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@57202fd
   [junit4]   2> 494082 T1856 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/jetty1
   [junit4]   2> 494082 T1856 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/jetty1/index/
   [junit4]   2> 494082 T1856 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 494083 T1856 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.BasicDistributedZk2Test-1392572548344/jetty1/index
   [junit4]   2> 494083 T1856 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=42, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 494084 T1856 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@15871 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@157e80c),segFN=segments_1,generation=1}
   [junit4]   2> 494084 T1856 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 494088 T1856 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 494088 T1856 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 494089 T1856 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 494089 T1856 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 494089 T1856 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 494089 T1856 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 494090 T1856 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 494090 T1856 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 494090 T1856 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 494091 T1856 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 494092 T1856 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 494092 T1856 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 494092 T1856 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 494093 T1856 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 494093 T1856 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 494094 T1856 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 494105 T1856 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 494109 T1856 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 494110 T1856 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 494110 T1856 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=66.173828125, floorSegmentMB=0.6552734375, forceMergeDeletesPctAllowed=19.629843639066507, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.14090802625859133
   [junit4]   2> 494111 T1856 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@15871 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@157e80c),segFN=segments_1,generation=1}
   [junit4]   2> 494111 T1856 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 494112 T1856 oass.SolrIndexSearcher.<init> Opening Searcher@54941a8b[collection1] main
   [junit4]   2> 494115 T1857 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54941a8b[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 494118 T1856 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 494119 T1809 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0
   [junit4]   2> 494119 T1809 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 494119 T1860 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:11374/naom/c collection:collection1 shard:shard1
   [junit4]   2> 494121 T1860 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 494147 T1860 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 494159 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494159 T1860 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 494159 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494159 T1860 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 494160 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494160 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C260 name=collection1 org.apache.solr.core.SolrCore@51696e47 url=https://127.0.0.1:11374/naom/c/collection1 node=127.0.0.1:11374_naom%2Fc C260_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:11374/naom/c, core=collection1, node_name=127.0.0.1:11374_naom%2Fc}
   [junit4]   2> 494160 T1860 C260 P11374 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:11374/naom/c/collection1/
   [junit4]   2> 494161 T1860 C260 P11374 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 494161 T1860 C260 P11374 oasc.SyncStrategy.syncToMe https://127.0.0.1:11374/naom/c/collection1/ has no replicas
   [junit4]   2> 494161 T1860 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:11374/naom/c/collection1/ shard1
   [junit4]   2> 494162 T1860 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 494169 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494170 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494170 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494170 T1855 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494175 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494176 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 494179 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494286 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494286 T1855 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494286 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494329 T1860 oasc.ZkController.register We are https://127.0.0.1:11374/naom/c/collection1/ and leader is https://127.0.0.1:11374/naom/c/collection1/
   [junit4]   2> 494329 T1860 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:11374/naom/c
   [junit4]   2> 494330 T1860 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 494330 T1860 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 494330 T1860 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 494332 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494332 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494332 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494332 T1860 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 494333 T1809 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 494333 T1834 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 494334 T1809 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 494334 T1834 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:11374/naom/c",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:11374_naom%2Fc",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 494337 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 494338 T1809 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 494340 T1809 oejs.AbstractConnector.doStart Started SslSocketConnector@127.0.0.1:11377
   [junit4]   2> 494343 T1809 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 494343 T1809 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 494344 T1809 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392572555144
   [junit4]   2> 494344 T1809 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392572555144/'
   [junit4]   2> 494376 T1809 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build/solr-core/test/J0/./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392572555144/solr.xml
   [junit4]   2> 494440 T1833 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494440 T1842 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494440 T1855 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 494446 T1809 oasc.CoreContainer.<init> New CoreContainer 735994700
   [junit4]   2> 494446 T1809 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.BasicDistributedZk2Test-jetty2-1392572555144/]
   [junit4]   2> 494447 T1809 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 494448 T1809 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 494448 T1809 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 494448 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 494449 T1809 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 494449 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 494449 T1809 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 494450 T1809 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 494450 T1809 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 494461 T1809 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 494461 T1809 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 494462 T1809 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 494462 T1809 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:11366/solr
   [junit4]   2> 494463 T1809 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 494463 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 494465 T1871 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8d4ef5f name:ZooKeeperConnection Watcher:127.0.0.1:11366 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 494466 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 494469 T1809 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 494476 T1873 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@711b9b2a name:ZooKeeperConnection Watcher:127.0.0.1:11366/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 494476 T1809 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 494482 T1809 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 495486 T1809 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:11377_naom%2Fc
   [junit4]   2> 495488 T1809 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:11377_naom%2Fc
   [junit4]   2> 495492 T1855 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 495493 T1873 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 495493 T1842 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 495492 T1833 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 495510 T1874 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 495510 T1874 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 495512 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 495512 T1874 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 495512 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 495513 T1833 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher f

[...truncated too long message...]

]   2> 2154995 T5801 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1899970091
   [junit4]   2> 2154996 T5801 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3a19db26
   [junit4]   2> 2155015 T5801 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=2,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=137,cumulative_deletesById=0,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=14499,transaction_logs_total_number=1}
   [junit4]   2> 2155016 T5801 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2155017 T5801 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2155017 T5801 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2155019 T5801 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2155020 T5801 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2155020 T5801 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.TestDistributedGrouping-1392574203404/shard1/data/index [CachedDir<<refCount=0;path=./org.apache.solr.TestDistributedGrouping-1392574203404/shard1/data/index;done=false>>]
   [junit4]   2> 2155021 T5801 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.TestDistributedGrouping-1392574203404/shard1/data/index
   [junit4]   2> 2155021 T5801 oasc.CachingDirectoryFactory.closeCacheValue looking to close ./org.apache.solr.TestDistributedGrouping-1392574203404/shard1/data [CachedDir<<refCount=0;path=./org.apache.solr.TestDistributedGrouping-1392574203404/shard1/data;done=false>>]
   [junit4]   2> 2155022 T5801 oasc.CachingDirectoryFactory.close Closing directory: ./org.apache.solr.TestDistributedGrouping-1392574203404/shard1/data
   [junit4]   2> 2155023 T5801 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2155120 T5801 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDistributedGrouping -Dtests.method=testDistribSearch -Dtests.seed=5D57ED374AC8547 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=ar_YE -Dtests.timezone=Asia/Dacca -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   12.7s J1 | TestDistributedGrouping.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]    > <head>
   [junit4]    > <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
   [junit4]    > <title>Error 500 Server Error</title>
   [junit4]    > </head>
   [junit4]    > <body>
   [junit4]    > <h2>HTTP ERROR: 500</h2>
   [junit4]    > <p>Problem accessing /select. Reason:
   [junit4]    > <pre>    Server Error</pre></p>
   [junit4]    > <hr /><i><small>Powered by Jetty://</small></i>
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    >                                                 
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5D57ED374AC8547:8433F0CB03F3E57B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:459)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:199)
   [junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:90)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:515)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:563)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:545)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:524)
   [junit4]    > 	at org.apache.solr.TestDistributedGrouping.doTest(TestDistributedGrouping.java:156)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:876)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 2155130 T5801 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Asserting, sim=DefaultSimilarity, locale=ar_YE, timezone=Asia/Dacca
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=2,free=234015576,total=535822336
   [junit4]   2> NOTE: All tests run in this JVM: [TestFieldCollectionResource, TestReversedWildcardFilterFactory, ChaosMonkeySafeLeaderTest, TestQueryUtils, CoreAdminCreateDiscoverTest, SynonymTokenizerTest, TestDefaultSearchFieldResource, SpellCheckComponentTest, TestCopyFieldCollectionResource, TestSweetSpotSimilarityFactory, UnloadDistributedZkTest, TestLRUCache, TestMaxScoreQueryParser, DistribCursorPagingTest, TestCustomSort, ChangedSchemaMergeTest, LukeRequestHandlerTest, SolrIndexSplitterTest, DOMUtilTest, TestSolrQueryParserResource, TestCodecSupport, ShowFileRequestHandlerTest, SpellCheckCollatorTest, SolrInfoMBeanTest, HighlighterTest, TestSystemIdResolver, TestFaceting, TestOmitPositions, JsonLoaderTest, ClusterStateUpdateTest, DocumentBuilderTest, SuggesterTest, TestRecoveryHdfs, TestBinaryResponseWriter, ScriptEngineTest, TestHashPartitioner, UpdateParamsTest, SpellingQueryConverterTest, TestPartialUpdateDeduplication, DeleteReplicaTest, TestWordDelimiterFilterFactory, CSVRequestHandlerTest, HdfsLockFactoryTest, RAMDirectoryFactoryTest, SimplePostToolTest, ConnectionManagerTest, TestOverriddenPrefixQueryForCustomFieldType, UUIDFieldTest, DocValuesMissingTest, SliceStateUpdateTest, TestLuceneMatchVersion, TestSolrXmlPersistence, DirectUpdateHandlerTest, RequestHandlersTest, RecoveryZkTest, TestSurroundQueryParser, TestStressVersions, TestFiltering, TestBM25SimilarityFactory, FullSolrCloudDistribCmdsTest, HdfsRecoveryZkTest, BJQParserTest, TestIndexingPerformance, TestCSVLoader, DirectSolrConnectionTest, TestPHPSerializedResponseWriter, BadComponentTest, TestShardHandlerFactory, TestQuerySenderNoQuery, TestJmxMonitoredMap, ChaosMonkeyNothingIsSafeTest, DistanceFunctionTest, ClusterStateTest, AnalysisAfterCoreReloadTest, DistributedDebugComponentTest, SolrRequestParserTest, ShardSplitTest, LeaderElectionIntegrationTest, LeaderElectionTest, TestRandomFaceting, DistributedSuggestComponentTest, FileBasedSpellCheckerTest, OverseerRolesTest, StressHdfsTest, XsltUpdateRequestHandlerTest, TestDocumentBuilder, TestDFRSimilarityFactory, PolyFieldTest, BasicZkTest, BasicDistributedZkTest, TestDistributedSearch, TriLevelCompositeIdRoutingTest, TestRandomDVFaceting, CustomCollectionTest, TestJoin, TestCloudManagedSchema, TestCloudManagedSchemaAddField, TestReload, OverseerTest, TestDistribDocBasedVersion, TestDistributedGrouping]
   [junit4] Completed on J1 in 12.71s, 1 test, 1 error <<< FAILURES!

[...truncated 518 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:459: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:439: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/solr/common-build.xml:491: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:1308: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-4.x-Java7/lucene/common-build.xml:941: There were test failures: 362 suites, 1563 tests, 4 errors, 35 ignored (7 assumptions)

Total time: 68 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure