You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2013/11/22 01:18:48 UTC

[JENKINS] Lucene-Solr-Tests-trunk-Java7 - Build # 4496 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-Tests-trunk-Java7/4496/

2 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.testDistribSearch

Error Message:
document count mismatch.  control=173 sum(shards)=174 cloudClient=174

Stack Trace:
java.lang.AssertionError: document count mismatch.  control=173 sum(shards)=174 cloudClient=174
	at __randomizedtesting.SeedInfo.seed([C733176CEA52D0E6:46D599749D0DB0DA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1238)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.doTest(ChaosMonkeyNothingIsSafeTest.java:195)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)


REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:40736/hh_dfu/ai

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Timeout occured while waiting response from server at: http://127.0.0.1:40736/hh_dfu/ai
	at __randomizedtesting.SeedInfo.seed([C733176CEA52D0E6:46D599749D0DB0DA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:491)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:197)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:139)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1559)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.access$600(RandomizedRunner.java:79)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:737)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:773)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:787)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:782)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:442)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:746)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:648)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:682)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:693)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:70)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:358)
	at java.lang.Thread.run(Thread.java:724)




Build Log:
[...truncated 10561 lines...]
   [junit4] Suite: org.apache.solr.cloud.SyncSliceTest
   [junit4]   2> 1330327 T1876 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /hh_dfu/ai
   [junit4]   2> 1330333 T1876 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> Creating dataDir: /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./solrtest-SyncSliceTest-1385078619374
   [junit4]   2> 1330335 T1876 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1330336 T1877 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1330437 T1876 oasc.ZkTestServer.run start zk server on port:49211
   [junit4]   2> 1330438 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1330466 T1883 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@57d6a20d name:ZooKeeperConnection Watcher:127.0.0.1:49211 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1330467 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1330467 T1876 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1330478 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1330480 T1885 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a7c25fb name:ZooKeeperConnection Watcher:127.0.0.1:49211/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1330480 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1330481 T1876 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1330491 T1876 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1330494 T1876 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1330496 T1876 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1330506 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1330507 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1330520 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1330521 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1330626 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1330627 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1330631 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1330631 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1330641 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1330642 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1330646 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1330647 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1330651 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1330652 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1330655 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1330656 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1330659 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1330660 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1330664 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1330665 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1330668 T1876 oasc.AbstractZkTestCase.putConfig put /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1330669 T1876 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1331046 T1876 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1331050 T1876 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:53581
   [junit4]   2> 1331051 T1876 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1331052 T1876 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1331052 T1876 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713
   [junit4]   2> 1331053 T1876 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/'
   [junit4]   2> 1331110 T1876 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/solr.xml
   [junit4]   2> 1331236 T1876 oasc.CoreContainer.<init> New CoreContainer 1165966071
   [junit4]   2> 1331237 T1876 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/]
   [junit4]   2> 1331240 T1876 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1331240 T1876 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1331241 T1876 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1331241 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1331241 T1876 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1331242 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1331242 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1331243 T1876 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1331243 T1876 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1331254 T1876 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1331255 T1876 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1331255 T1876 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1331256 T1876 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49211/solr
   [junit4]   2> 1331256 T1876 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1331258 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1331286 T1896 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f8b1557 name:ZooKeeperConnection Watcher:127.0.0.1:49211 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1331288 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1331300 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1331302 T1898 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77189736 name:ZooKeeperConnection Watcher:127.0.0.1:49211/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1331303 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1331305 T1876 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1331315 T1876 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1331318 T1876 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1331325 T1876 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:53581_hh_dfu%2Fai
   [junit4]   2> 1331327 T1876 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:53581_hh_dfu%2Fai
   [junit4]   2> 1331330 T1876 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1331347 T1876 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1331351 T1876 oasc.Overseer.start Overseer (id=90772512399818755-127.0.0.1:53581_hh_dfu%2Fai-n_0000000000) starting
   [junit4]   2> 1331363 T1876 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1331375 T1900 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1331376 T1876 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1331379 T1876 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1331381 T1876 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1331383 T1899 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1331391 T1901 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1331392 T1901 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1331393 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1331393 T1901 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1331395 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1331396 T1899 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53581/hh_dfu/ai",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53581_hh_dfu%2Fai",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1331396 T1899 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1331396 T1899 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1331399 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1331408 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1332398 T1901 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1332399 T1901 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/collection1
   [junit4]   2> 1332399 T1901 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1332400 T1901 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1332400 T1901 oasc.ZkController.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1332402 T1901 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/collection1/'
   [junit4]   2> 1332404 T1901 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/collection1/lib/classes/' to classloader
   [junit4]   2> 1332405 T1901 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/collection1/lib/README' to classloader
   [junit4]   2> 1332478 T1901 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1332548 T1901 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1332649 T1901 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1332669 T1901 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1333340 T1901 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1333352 T1901 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1333355 T1901 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1333366 T1901 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1333392 T1901 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1333397 T1901 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1333398 T1901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1333398 T1901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1333398 T1901 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1333399 T1901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1333400 T1901 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1333400 T1901 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1333400 T1901 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-controljetty-1385078619713/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1385078619374/control/data/
   [junit4]   2> 1333400 T1901 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@45c69e8
   [junit4]   2> 1333401 T1901 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.SyncSliceTest-1385078619374/control/data
   [junit4]   2> 1333401 T1901 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1385078619374/control/data/index/
   [junit4]   2> 1333402 T1901 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1385078619374/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1333402 T1901 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.SyncSliceTest-1385078619374/control/data/index
   [junit4]   2> 1333402 T1901 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1333403 T1901 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7e4cc78c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@29aa07a8),segFN=segments_1,generation=1}
   [junit4]   2> 1333403 T1901 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1333407 T1901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1333407 T1901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1333407 T1901 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1333408 T1901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1333408 T1901 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1333408 T1901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1333409 T1901 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1333409 T1901 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1333409 T1901 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1333410 T1901 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1333410 T1901 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1333410 T1901 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1333410 T1901 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1333411 T1901 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1333411 T1901 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1333412 T1901 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1333422 T1901 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1333427 T1901 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1333427 T1901 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1333428 T1901 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1333428 T1901 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@7e4cc78c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@29aa07a8),segFN=segments_1,generation=1}
   [junit4]   2> 1333428 T1901 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1333429 T1901 oass.SolrIndexSearcher.<init> Opening Searcher@a3b0996 main
   [junit4]   2> 1333431 T1902 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@a3b0996 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1333432 T1901 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1333432 T1901 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:53581/hh_dfu/ai collection:control_collection shard:shard1
   [junit4]   2> 1333434 T1901 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1333443 T1901 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1333460 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1333460 T1901 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1333460 T1901 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C870 name=collection1 org.apache.solr.core.SolrCore@6ed0ccc2 url=http://127.0.0.1:53581/hh_dfu/ai/collection1 node=127.0.0.1:53581_hh_dfu%2Fai C870_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:53581/hh_dfu/ai, core=collection1, node_name=127.0.0.1:53581_hh_dfu%2Fai}
   [junit4]   2> 1333461 T1901 C870 P53581 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:53581/hh_dfu/ai/collection1/
   [junit4]   2> 1333461 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1333461 T1901 C870 P53581 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1333462 T1901 C870 P53581 oasc.SyncStrategy.syncToMe http://127.0.0.1:53581/hh_dfu/ai/collection1/ has no replicas
   [junit4]   2> 1333462 T1901 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:53581/hh_dfu/ai/collection1/ shard1
   [junit4]   2> 1333462 T1901 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1333464 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1333465 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1333469 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1333472 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1333574 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1333622 T1901 oasc.ZkController.register We are http://127.0.0.1:53581/hh_dfu/ai/collection1/ and leader is http://127.0.0.1:53581/hh_dfu/ai/collection1/
   [junit4]   2> 1333622 T1901 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:53581/hh_dfu/ai
   [junit4]   2> 1333622 T1901 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1333623 T1901 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1333623 T1901 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1333624 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1333625 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1333625 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1333625 T1901 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1333626 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1333626 T1876 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1333627 T1876 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1333627 T1899 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:53581/hh_dfu/ai",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:53581_hh_dfu%2Fai",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1333630 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1333630 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1333638 T1905 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@497e83ec name:ZooKeeperConnection Watcher:127.0.0.1:49211/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1333638 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1333639 T1876 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1333642 T1876 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1333733 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1333733 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1333933 T1876 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1333936 T1876 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:40736
   [junit4]   2> 1333937 T1876 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1333938 T1876 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1333938 T1876 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682
   [junit4]   2> 1333939 T1876 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/'
   [junit4]   2> 1333980 T1876 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/solr.xml
   [junit4]   2> 1334076 T1876 oasc.CoreContainer.<init> New CoreContainer 102663315
   [junit4]   2> 1334077 T1876 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/]
   [junit4]   2> 1334079 T1876 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1334079 T1876 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1334080 T1876 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1334080 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1334081 T1876 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1334081 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1334082 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1334082 T1876 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1334083 T1876 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1334090 T1876 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1334090 T1876 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1334091 T1876 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1334091 T1876 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49211/solr
   [junit4]   2> 1334092 T1876 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1334093 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1334096 T1916 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1aa5cfee name:ZooKeeperConnection Watcher:127.0.0.1:49211 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1334096 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1334100 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1334102 T1918 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66e722ae name:ZooKeeperConnection Watcher:127.0.0.1:49211/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1334102 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1334113 T1876 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1335117 T1876 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40736_hh_dfu%2Fai
   [junit4]   2> 1335146 T1876 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40736_hh_dfu%2Fai
   [junit4]   2> 1335152 T1898 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1335153 T1918 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1335153 T1905 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1335172 T1919 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1335173 T1919 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1335175 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1335176 T1919 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1335176 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1335177 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1335178 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1335179 T1899 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40736/hh_dfu/ai",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40736_hh_dfu%2Fai",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1335180 T1899 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 1335180 T1899 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1335185 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1335319 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1335320 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1335320 T1918 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1336177 T1919 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1336177 T1919 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/collection1
   [junit4]   2> 1336178 T1919 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1336179 T1919 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1336180 T1919 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1336181 T1919 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/collection1/'
   [junit4]   2> 1336185 T1919 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/collection1/lib/README' to classloader
   [junit4]   2> 1336186 T1919 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/collection1/lib/classes/' to classloader
   [junit4]   2> 1336317 T1919 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1336509 T1919 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1336611 T1919 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1336656 T1919 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1337643 T1919 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1337655 T1919 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1337660 T1919 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1337675 T1919 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1337680 T1919 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1337686 T1919 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1337687 T1919 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1337689 T1919 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1337689 T1919 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1337690 T1919 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1337690 T1919 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1337691 T1919 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1337691 T1919 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.SyncSliceTest-jetty1-1385078622682/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty1/
   [junit4]   2> 1337691 T1919 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@45c69e8
   [junit4]   2> 1337692 T1919 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty1
   [junit4]   2> 1337693 T1919 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty1/index/
   [junit4]   2> 1337693 T1919 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1337693 T1919 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty1/index
   [junit4]   2> 1337694 T1919 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1337695 T1919 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@126630b9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@381a44e),segFN=segments_1,generation=1}
   [junit4]   2> 1337695 T1919 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1337701 T1919 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1337702 T1919 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1337702 T1919 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1337702 T1919 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1337703 T1919 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1337703 T1919 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1337704 T1919 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1337704 T1919 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1337704 T1919 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1337705 T1919 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1337706 T1919 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1337706 T1919 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1337706 T1919 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1337707 T1919 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1337708 T1919 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1337708 T1919 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1337723 T1919 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1337729 T1919 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1337729 T1919 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1337730 T1919 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1337731 T1919 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@126630b9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@381a44e),segFN=segments_1,generation=1}
   [junit4]   2> 1337731 T1919 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1337732 T1919 oass.SolrIndexSearcher.<init> Opening Searcher@764323cf main
   [junit4]   2> 1337737 T1920 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@764323cf main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1337740 T1919 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1337741 T1919 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:40736/hh_dfu/ai collection:collection1 shard:shard1
   [junit4]   2> 1337742 T1919 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1337969 T1919 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1338026 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338026 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338026 T1919 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1338027 T1919 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1338026 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C871 name=collection1 org.apache.solr.core.SolrCore@30b598f0 url=http://127.0.0.1:40736/hh_dfu/ai/collection1 node=127.0.0.1:40736_hh_dfu%2Fai C871_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:40736/hh_dfu/ai, core=collection1, node_name=127.0.0.1:40736_hh_dfu%2Fai}
   [junit4]   2> 1338027 T1919 C871 P40736 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:40736/hh_dfu/ai/collection1/
   [junit4]   2> 1338028 T1919 C871 P40736 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1338028 T1919 C871 P40736 oasc.SyncStrategy.syncToMe http://127.0.0.1:40736/hh_dfu/ai/collection1/ has no replicas
   [junit4]   2> 1338028 T1919 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:40736/hh_dfu/ai/collection1/ shard1
   [junit4]   2> 1338028 T1919 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1338029 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1338045 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338055 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338055 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338055 T1918 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338070 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1338079 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338191 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338191 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338191 T1918 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338222 T1919 oasc.ZkController.register We are http://127.0.0.1:40736/hh_dfu/ai/collection1/ and leader is http://127.0.0.1:40736/hh_dfu/ai/collection1/
   [junit4]   2> 1338222 T1919 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:40736/hh_dfu/ai
   [junit4]   2> 1338222 T1919 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1338222 T1919 oasc.ZkController.publish publishing core=collection1 state=active
   [junit4]   2> 1338223 T1919 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1338229 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338229 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338229 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338229 T1919 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1338231 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1338231 T1876 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1338232 T1876 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1338231 T1899 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40736/hh_dfu/ai",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40736_hh_dfu%2Fai",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1338247 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1338349 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338350 T1918 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338350 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1338544 T1876 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1338548 T1876 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:40745
   [junit4]   2> 1338549 T1876 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1338549 T1876 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1338550 T1876 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273
   [junit4]   2> 1338550 T1876 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/'
   [junit4]   2> 1338592 T1876 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/solr.xml
   [junit4]   2> 1338688 T1876 oasc.CoreContainer.<init> New CoreContainer 1611214157
   [junit4]   2> 1338689 T1876 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/]
   [junit4]   2> 1338691 T1876 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1338692 T1876 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1338692 T1876 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1338693 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1338693 T1876 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1338694 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1338694 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1338694 T1876 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1338695 T1876 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1338702 T1876 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1338702 T1876 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1338703 T1876 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1338704 T1876 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49211/solr
   [junit4]   2> 1338704 T1876 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1338705 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1338709 T1932 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1cd2d30d name:ZooKeeperConnection Watcher:127.0.0.1:49211 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1338710 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1338721 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1338728 T1934 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bcee13d name:ZooKeeperConnection Watcher:127.0.0.1:49211/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1338728 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1338748 T1876 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1339752 T1876 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40745_hh_dfu%2Fai
   [junit4]   2> 1339759 T1876 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40745_hh_dfu%2Fai
   [junit4]   2> 1339763 T1898 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1339763 T1918 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1339763 T1934 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1339763 T1905 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1339778 T1935 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1339778 T1935 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1339780 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1339780 T1935 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1339780 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1339780 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1339781 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1339782 T1899 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40745/hh_dfu/ai",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40745_hh_dfu%2Fai",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1339782 T1899 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 1339782 T1899 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1339785 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1339888 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1339888 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1339888 T1934 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1339888 T1918 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1340781 T1935 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1340782 T1935 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ./org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/collection1
   [junit4]   2> 1340782 T1935 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1340785 T1935 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1340786 T1935 oasc.ZkController.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1340791 T1935 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/collection1/'
   [junit4]   2> 1340796 T1935 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/collection1/lib/README' to classloader
   [junit4]   2> 1340798 T1935 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/collection1/lib/classes/' to classloader
   [junit4]   2> 1341007 T1935 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_50
   [junit4]   2> 1341223 T1935 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1341325 T1935 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1341363 T1935 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1343022 T1935 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1343071 T1935 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1343081 T1935 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1343116 T1935 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1343129 T1935 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1343140 T1935 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1343143 T1935 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1343145 T1935 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1343146 T1935 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1343147 T1935 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1343148 T1935 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1343148 T1935 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1343149 T1935 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./org.apache.solr.cloud.SyncSliceTest-jetty2-1385078627273/collection1/, dataDir=./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty2/
   [junit4]   2> 1343149 T1935 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@45c69e8
   [junit4]   2> 1343151 T1935 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty2
   [junit4]   2> 1343151 T1935 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty2/index/
   [junit4]   2> 1343151 T1935 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1343152 T1935 oasc.CachingDirectoryFactory.get return new directory for ./org.apache.solr.cloud.SyncSliceTest-1385078619374/jetty2/index
   [junit4]   2> 1343152 T1935 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 1343154 T1935 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a29c160 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5efb2740),segFN=segments_1,generation=1}
   [junit4]   2> 1343154 T1935 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1343162 T1935 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1343163 T1935 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1343163 T1935 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1343163 T1935 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1343164 T1935 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1343164 T1935 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1343165 T1935 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1343165 T1935 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1343166 T1935 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1343167 T1935 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1343168 T1935 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1343168 T1935 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1343169 T1935 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1343170 T1935 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1343170 T1935 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1343171 T1935 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1343206 T1935 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1343212 T1935 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1343212 T1935 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1343213 T1935 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=25, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 1343214 T1935 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(org.apache.lucene.store.RAMDirectory@4a29c160 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5efb2740),segFN=segments_1,generation=1}
   [junit4]   2> 1343214 T1935 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1343214 T1935 oass.SolrIndexSearcher.<init> Opening Searcher@22fdd72 main
   [junit4]   2> 1343220 T1936 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@22fdd72 main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1343224 T1935 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1343225 T1935 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:40745/hh_dfu/ai collection:collection1 shard:shard1
   [junit4]   2> 1343233 T1935 oasc.ZkController.register We are http://127.0.0.1:40745/hh_dfu/ai/collection1/ and leader is http://127.0.0.1:40736/hh_dfu/ai/collection1/
   [junit4]   2> 1343234 T1935 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:40745/hh_dfu/ai
   [junit4]   2> 1343234 T1935 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 1343234 T1935 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C872 name=collection1 org.apache.solr.core.SolrCore@2d62ac19 url=http://127.0.0.1:40745/hh_dfu/ai/collection1 node=127.0.0.1:40745_hh_dfu%2Fai C872_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:40745/hh_dfu/ai, core=collection1, node_name=127.0.0.1:40745_hh_dfu%2Fai}
   [junit4]   2> 1343236 T1937 C872 P40745 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 1343237 T1937 C872 P40745 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 1343237 T1935 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1343237 T1937 C872 P40745 oasc.ZkController.publish publishing core=collection1 state=recovering
   [junit4]   2> 1343238 T1937 C872 P40745 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1343240 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1343240 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1343240 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1343241 T1876 oass.SolrDispatchFilter.init user.dir=/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1
   [junit4]   2> 1343242 T1876 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1343243 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1343244 T1908 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true
   [junit4]   2> 1343244 T1899 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40745/hh_dfu/ai",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40745_hh_dfu%2Fai",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1343257 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1343259 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1343259 T1934 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1343259 T1918 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1343259 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1343560 T1876 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1343562 T1876 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:40774
   [junit4]   2> 1343563 T1876 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1343564 T1876 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1343564 T1876 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-jetty3-1385078632285
   [junit4]   2> 1343564 T1876 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-jetty3-1385078632285/'
   [junit4]   2> 1343606 T1876 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-jetty3-1385078632285/solr.xml
   [junit4]   2> 1343880 T1876 oasc.CoreContainer.<init> New CoreContainer 588380238
   [junit4]   2> 1343881 T1876 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../../usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build/solr-core/test/J1/./org.apache.solr.cloud.SyncSliceTest-jetty3-1385078632285/]
   [junit4]   2> 1343883 T1876 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1343883 T1876 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: http://
   [junit4]   2> 1343884 T1876 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1343884 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1343884 T1876 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1343885 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1343885 T1876 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1343885 T1876 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1343886 T1876 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1343895 T1876 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1343896 T1876 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1343896 T1876 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1343897 T1876 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:49211/solr
   [junit4]   2> 1343897 T1876 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1343898 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1343907 T1949 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2751af65 name:ZooKeeperConnection Watcher:127.0.0.1:49211 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1343908 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1343918 T1876 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1343920 T1951 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2b203c1c name:ZooKeeperConnection Watcher:127.0.0.1:49211/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1343921 T1876 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1343933 T1876 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1344245 T1908 oasha.CoreAdminHandler.handleWaitForStateAction Waited coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true for: 1 seconds.
   [junit4]   2> 1344245 T1908 oass.SolrDispatchFilter.handleAdminRequest [admin] webapp=null path=/admin/cores params={coreNodeName=core_node2&state=recovering&nodeName=127.0.0.1:40745_hh_dfu%252Fai&action=PREPRECOVERY&checkLive=true&core=collection1&wt=javabin&onlyIfLeader=true&version=2} status=0 QTime=1001 
   [junit4]   2> 1344937 T1876 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40774_hh_dfu%2Fai
   [junit4]   2> 1344939 T1876 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40774_hh_dfu%2Fai
   [junit4]   2> 1344943 T1905 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1344944 T1951 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1344944 T1918 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1344944 T1898 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1344944 T1934 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1344953 T1952 oasc.ZkController.publish publishing core=collection1 state=down
   [junit4]   2> 1344953 T1952 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1344955 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1344955 T1952 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1344956 T1899 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1344957 T1899 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40774/hh_dfu/ai",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40774_hh_dfu%2Fai",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "shard_range":null,
   [junit4]   2> 	  "shard_state":"active",
   [junit4]   2> 	  "shard_parent":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1344957 T1899 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 1344958 T1899 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1344961 T1898 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1344968 T1898 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1344968 T1918 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1344968 T1905 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1344968 T1934 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1344968 T1951 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1345956 T1952 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1345956 T1952 oasc.CoreContainer.create Creating SolrCore 'collection1' using instanceDir: ../../../../../../../../../../../usr/home/hudson/hudson-slave/worksp

[...truncated too long message...]

  [junit4]   1>       /solr/collections/collection1/leader_elect/shard2/election (3)
   [junit4]   1>        /solr/collections/collection1/leader_elect/shard2/election/90772555074699284-core_node2-n_0000000003 (0)
   [junit4]   1>        /solr/collections/collection1/leader_elect/shard2/election/90772555074699276-core_node4-n_0000000001 (0)
   [junit4]   1>        /solr/collections/collection1/leader_elect/shard2/election/90772555074699280-core_node6-n_0000000002 (0)
   [junit4]   1>     /solr/collections/collection1/leaders (2)
   [junit4]   1>      /solr/collections/collection1/leaders/shard1 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "core":"collection1",
   [junit4]   1>            "node_name":"127.0.0.1:10278_",
   [junit4]   1>            "base_url":"http://127.0.0.1:10278"}
   [junit4]   1>      /solr/collections/collection1/leaders/shard2 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "core":"collection1",
   [junit4]   1>            "node_name":"127.0.0.1:16657_",
   [junit4]   1>            "base_url":"http://127.0.0.1:16657"}
   [junit4]   1>    /solr/collections/control_collection (3)
   [junit4]   1>    DATA:
   [junit4]   1>        {"configName":"conf1"}
   [junit4]   1>     /solr/collections/control_collection/shards (0)
   [junit4]   1>     /solr/collections/control_collection/leader_elect (1)
   [junit4]   1>      /solr/collections/control_collection/leader_elect/shard1 (1)
   [junit4]   1>       /solr/collections/control_collection/leader_elect/shard1/election (1)
   [junit4]   1>        /solr/collections/control_collection/leader_elect/shard1/election/90772555074699267-core_node1-n_0000000000 (0)
   [junit4]   1>     /solr/collections/control_collection/leaders (1)
   [junit4]   1>      /solr/collections/control_collection/leaders/shard1 (0)
   [junit4]   1>      DATA:
   [junit4]   1>          {
   [junit4]   1>            "core":"collection1",
   [junit4]   1>            "node_name":"127.0.0.1:56834_",
   [junit4]   1>            "base_url":"http://127.0.0.1:56834"}
   [junit4]   1>   /solr/overseer_elect (2)
   [junit4]   1>    /solr/overseer_elect/election (7)
   [junit4]   1>     /solr/overseer_elect/election/90772555074699284-127.0.0.1:15079_-n_0000000008 (0)
   [junit4]   1>     /solr/overseer_elect/election/90772555074699278-127.0.0.1:50970_-n_0000000005 (0)
   [junit4]   1>     /solr/overseer_elect/election/90772555074699267-127.0.0.1:56834_-n_0000000000 (0)
   [junit4]   1>     /solr/overseer_elect/election/90772555074699276-127.0.0.1:16657_-n_0000000004 (0)
   [junit4]   1>     /solr/overseer_elect/election/90772555074699270-127.0.0.1:10278_-n_0000000001 (0)
   [junit4]   1>     /solr/overseer_elect/election/90772555074699274-127.0.0.1:48324_-n_0000000003 (0)
   [junit4]   1>     /solr/overseer_elect/election/90772555074699280-127.0.0.1:35765_-n_0000000006 (0)
   [junit4]   1>    /solr/overseer_elect/leader (0)
   [junit4]   1>    DATA:
   [junit4]   1>        {"id":"90772555074699267-127.0.0.1:56834_-n_0000000000"}
   [junit4]   1>  /zookeeper (1)
   [junit4]   1>  DATA:
   [junit4]   1>      
   [junit4]   1> 
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeyNothingIsSafeTest -Dtests.method=testDistribSearch -Dtests.seed=C733176CEA52D0E6 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=be_BY -Dtests.timezone=Atlantic/Faeroe -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 98.5s J1 | ChaosMonkeyNothingIsSafeTest.testDistribSearch <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: document count mismatch.  control=173 sum(shards)=174 cloudClient=174
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C733176CEA52D0E6:46D599749D0DB0DA]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1238)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.doTest(ChaosMonkeyNothingIsSafeTest.java:195)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:835)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:724)
   [junit4]   2> 2080024 T3391 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 98535 T3390 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 2080188 T3518 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> NOTE: test params are: codec=Lucene46: {timestamp=PostingsFormat(name=FSTOrdPulsing41), range_facet_l=PostingsFormat(name=FSTOrdPulsing41), id=Lucene41(blocksize=128), range_facet_sl=Lucene41(blocksize=128), a_t=PostingsFormat(name=FSTOrdPulsing41), text=PostingsFormat(name=Lucene41VarGapFixedInterval), range_facet_si=PostingsFormat(name=FSTOrdPulsing41), _version_=PostingsFormat(name=Lucene41WithOrds), other_tl1=PostingsFormat(name=Lucene41WithOrds), multiDefault=PostingsFormat(name=FSTOrdPulsing41), a_si=PostingsFormat(name=Lucene41WithOrds), intDefault=PostingsFormat(name=Lucene41WithOrds)}, docValues:{timestamp=DocValuesFormat(name=Disk)}, sim=DefaultSimilarity, locale=be_BY, timezone=Atlantic/Faeroe
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_25 (64-bit)/cpus=16,threads=1,free=61808096,total=536870912
   [junit4]   2> NOTE: All tests run in this JVM: [AbstractAnalyticsStatsTest, TestReload, SolrCoreCheckLockOnStartupTest, DisMaxRequestHandlerTest, TestQuerySenderListener, TestRecovery, NumericFieldsTest, DocValuesMultiTest, TestAddFieldRealTimeGet, TestBinaryField, TestIndexingPerformance, TestPhraseSuggestions, TestFastWriter, StandardRequestHandlerTest, TestOmitPositions, ShardSplitTest, SimplePostToolTest, CoreContainerCoreInitFailuresTest, TestDistribDocBasedVersion, HdfsBasicDistributedZk2Test, TermVectorComponentDistributedTest, SolrPluginUtilsTest, TestRangeQuery, TestDistributedSearch, TimeZoneUtilsTest, CurrencyFieldXmlFileTest, AbstractAnalyticsFacetTest, HardAutoCommitTest, PrimUtilsTest, HdfsLockFactoryTest, NoFacetTest, ExpressionTest, CurrencyFieldOpenExchangeTest, MultiTermTest, TestStressRecovery, HighlighterTest, HighlighterConfigTest, BasicDistributedZk2Test, DirectSolrSpellCheckerTest, TestSolrQueryParserDefaultOperatorResource, SpellCheckComponentTest, DefaultValueUpdateProcessorTest, PluginInfoTest, TestSolrJ, RequestHandlersTest, TestSimpleQParserPlugin, IndexSchemaRuntimeFieldTest, FieldAnalysisRequestHandlerTest, TestStressReorder, DeleteInactiveReplicaTest, MinimalSchemaTest, CachingDirectoryFactoryTest, TestIBSimilarityFactory, BadIndexSchemaTest, ZkSolrClientTest, TestFastLRUCache, SuggesterWFSTTest, BasicZkTest, DistanceFunctionTest, TestIndexSearcher, DocumentAnalysisRequestHandlerTest, TestArbitraryIndexDir, AssignTest, TestWordDelimiterFilterFactory, FunctionTest, SuggesterTSTTest, AlternateDirectoryTest, FullSolrCloudDistribCmdsTest, TestSweetSpotSimilarityFactory, TestFieldResource, DirectUpdateHandlerTest, TestSchemaVersionResource, TestJmxIntegration, SolrRequestParserTest, TestSolrXml, TestFuzzyAnalyzedSuggestions, TestStressLucene, XsltUpdateRequestHandlerTest, OutputWriterTest, TestHashPartitioner, TestZkChroot, SyncSliceTest, LeaderElectionIntegrationTest, TestReplicationHandler, ClusterStateUpdateTest, TestRandomDVFaceting, TestFaceting, TestMultiCoreConfBootstrap, TestCoreContainer, TestSolr4Spatial, StatsComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, PeerSyncTest, ConvertedLegacyTest, TestSort, TestFiltering, SoftAutoCommitTest, SpellCheckCollatorTest, TestTrie, NoCacheHeaderTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, TestSolrDeletionPolicy1, CacheHeaderTest, LukeRequestHandlerTest, TermsComponentTest, TestValueSourceCache, RequiredFieldsTest, ReturnFieldsTest, MBeansHandlerTest, UniqFieldsUpdateProcessorFactoryTest, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, SearchHandlerTest, TestLFUCache, TestSolrIndexConfig, ResponseLogComponentTest, SolrIndexConfigTest, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, TestConfig, NotRequiredUniqueKeyTest, TestCodecSupport, TestXIncludeConfig, TestDFRSimilarityFactory, TestNumberUtils, TestLMJelinekMercerSimilarityFactory, TestDefaultSimilarityFactory, OpenExchangeRatesOrgProviderTest, ChaosMonkeyNothingIsSafeTest]
   [junit4] Completed on J1 in 98.82s, 1 test, 1 failure <<< FAILURES!

[...truncated 121 lines...]
BUILD FAILED
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:420: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:400: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/build.xml:39: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/extra-targets.xml:37: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/build.xml:189: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/solr/common-build.xml:489: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:1272: The following error occurred while executing this line:
/usr/home/hudson/hudson-slave/workspace/Lucene-Solr-Tests-trunk-Java7/lucene/common-build.xml:908: There were test failures: 349 suites, 1551 tests, 1 error, 1 failure, 34 ignored (6 assumptions)

Total time: 62 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure