You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/05/23 09:40:33 UTC

[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_20-ea-b11) - Build # 10244 - Still Failing!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10244/
Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops -XX:+UseG1GC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Leader never changed

Stack Trace:
java.lang.AssertionError: Leader never changed
	at __randomizedtesting.SeedInfo.seed([59A9E18F6772250E:D84F6F97102D4532]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:855)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.testDistribSearch

Error Message:
shard3 is not consistent.  Got 127 from http://127.0.0.1:35599/collection1lastClient and got 119 from http://127.0.0.1:45974/collection1

Stack Trace:
java.lang.AssertionError: shard3 is not consistent.  Got 127 from http://127.0.0.1:35599/collection1lastClient and got 119 from http://127.0.0.1:45974/collection1
	at __randomizedtesting.SeedInfo.seed([59A9E18F6772250E:D84F6F97102D4532]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1139)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.doTest(ChaosMonkeySafeLeaderTest.java:150)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11292 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/init-core-data-001
   [junit4]   2> 634127 T3416 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /h_j/d
   [junit4]   2> 634130 T3416 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 634130 T3416 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 634131 T3417 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 634231 T3416 oasc.ZkTestServer.run start zk server on port:41552
   [junit4]   2> 634232 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 634305 T3423 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3bfa3035 name:ZooKeeperConnection Watcher:127.0.0.1:41552 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 634306 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 634306 T3416 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 634313 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 634314 T3425 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5403d5fc name:ZooKeeperConnection Watcher:127.0.0.1:41552/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 634314 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 634315 T3416 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 634316 T3416 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 634318 T3416 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 634319 T3416 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 634320 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 634321 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 634323 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 634324 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 634326 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 634326 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 634330 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 634331 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 634332 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 634332 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 634334 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 634334 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 634336 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 634336 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 634337 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 634338 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 634339 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 634340 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 634342 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 634343 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 634344 T3416 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 634345 T3416 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 634636 T3416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 634640 T3416 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:34900
   [junit4]   2> 634640 T3416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 634641 T3416 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 634641 T3416 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002
   [junit4]   2> 634641 T3416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/'
   [junit4]   2> 634654 T3416 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/solr.xml
   [junit4]   2> 634662 T3416 oasc.CoreContainer.<init> New CoreContainer 1386545870
   [junit4]   2> 634662 T3416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/]
   [junit4]   2> 634662 T3416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 634663 T3416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 634663 T3416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 634663 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 634663 T3416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 634664 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 634664 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 634664 T3416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 634664 T3416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 634665 T3416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 634665 T3416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 634665 T3416 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 634666 T3416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41552/solr
   [junit4]   2> 634666 T3416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 634667 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 634669 T3436 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6737f9ba name:ZooKeeperConnection Watcher:127.0.0.1:41552 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 634669 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 634671 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 634672 T3438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14e33a71 name:ZooKeeperConnection Watcher:127.0.0.1:41552/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 634672 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 634673 T3416 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 634675 T3416 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 634676 T3416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 634678 T3416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 634680 T3416 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 634685 T3416 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 634686 T3416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34900_h_j%2Fd
   [junit4]   2> 634686 T3416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34900_h_j%2Fd
   [junit4]   2> 634687 T3416 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 634688 T3416 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 634692 T3416 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34900_h_j%2Fd
   [junit4]   2> 634692 T3416 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 634694 T3416 oasc.Overseer.start Overseer (id=91804781285081091-127.0.0.1:34900_h_j%2Fd-n_0000000000) starting
   [junit4]   2> 634696 T3416 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 634701 T3440 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 634701 T3416 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 634703 T3416 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 634704 T3416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 634705 T3439 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 634706 T3441 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 634707 T3441 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 634707 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 634707 T3441 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 634708 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 634708 T3439 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34900/h_j/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34900_h_j%2Fd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 634708 T3439 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 634709 T3439 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 634710 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 634710 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 635708 T3441 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 635708 T3441 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 635708 T3441 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 635709 T3441 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 635709 T3441 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 635709 T3441 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/collection1/'
   [junit4]   2> 635710 T3441 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 635710 T3441 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 635710 T3441 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 635728 T3441 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 635736 T3441 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 635737 T3441 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 635741 T3441 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 635844 T3441 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 635844 T3441 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 635844 T3441 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 635848 T3441 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 635849 T3441 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 635856 T3441 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 635858 T3441 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 635859 T3441 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 635860 T3441 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 635860 T3441 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 635860 T3441 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 635861 T3441 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 635861 T3441 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 635861 T3441 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 635861 T3441 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 635861 T3441 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-002/collection1/, dataDir=./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/control/data/
   [junit4]   2> 635862 T3441 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@32e944e7
   [junit4]   2> 635862 T3441 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/control/data
   [junit4]   2> 635862 T3441 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/control/data/index/
   [junit4]   2> 635862 T3441 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 635863 T3441 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/control/data/index
   [junit4]   2> 635863 T3441 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 635863 T3441 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@500bb693 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@195c195c),segFN=segments_1,generation=1}
   [junit4]   2> 635864 T3441 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 635865 T3441 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 635866 T3441 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 635866 T3441 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 635866 T3441 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 635866 T3441 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 635866 T3441 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 635867 T3441 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 635867 T3441 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 635867 T3441 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 635867 T3441 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 635867 T3441 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 635868 T3441 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 635868 T3441 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 635868 T3441 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 635868 T3441 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 635868 T3441 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 635870 T3441 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 635871 T3441 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 635871 T3441 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 635871 T3441 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=75.2890625, floorSegmentMB=1.884765625, forceMergeDeletesPctAllowed=16.58219904186481, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 635872 T3441 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@500bb693 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@195c195c),segFN=segments_1,generation=1}
   [junit4]   2> 635872 T3441 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 635872 T3441 oass.SolrIndexSearcher.<init> Opening Searcher@329cb109[collection1] main
   [junit4]   2> 635872 T3441 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 635873 T3441 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 635873 T3441 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 635873 T3441 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 635874 T3441 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 635874 T3441 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 635874 T3441 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 635874 T3441 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 635874 T3441 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 635874 T3441 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 635876 T3442 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@329cb109[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 635876 T3441 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 635877 T3445 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:34900/h_j/d collection:control_collection shard:shard1
   [junit4]   2> 635877 T3416 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 635877 T3416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 635877 T3445 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 635878 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 635879 T3447 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3068d44 name:ZooKeeperConnection Watcher:127.0.0.1:41552/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 635879 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 635880 T3416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 635881 T3445 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 635881 T3416 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 635884 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 635885 T3445 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 635885 T3445 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 635886 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1057 name=collection1 org.apache.solr.core.SolrCore@66434e05 url=http://127.0.0.1:34900/h_j/d/collection1 node=127.0.0.1:34900_h_j%2Fd C1057_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:34900/h_j/d, core=collection1, node_name=127.0.0.1:34900_h_j%2Fd}
   [junit4]   2> 635886 T3445 C1057 P34900 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:34900/h_j/d/collection1/
   [junit4]   2> 635886 T3445 C1057 P34900 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 635886 T3445 C1057 P34900 oasc.SyncStrategy.syncToMe http://127.0.0.1:34900/h_j/d/collection1/ has no replicas
   [junit4]   2> 635886 T3445 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:34900/h_j/d/collection1/ shard1
   [junit4]   2> 635887 T3445 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 635887 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 635890 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 635894 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 635995 T3447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 635995 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 636040 T3445 oasc.ZkController.register We are http://127.0.0.1:34900/h_j/d/collection1/ and leader is http://127.0.0.1:34900/h_j/d/collection1/
   [junit4]   2> 636041 T3445 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:34900/h_j/d
   [junit4]   2> 636041 T3445 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 636041 T3445 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 636041 T3445 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 636042 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 636042 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 636042 T3445 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 636042 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 636043 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 636044 T3439 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34900/h_j/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34900_h_j%2Fd",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 636045 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 636117 T3416 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 636118 T3416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 636120 T3416 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:42673
   [junit4]   2> 636121 T3416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 636121 T3416 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 636122 T3416 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003
   [junit4]   2> 636122 T3416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/'
   [junit4]   2> 636138 T3416 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/solr.xml
   [junit4]   2> 636145 T3416 oasc.CoreContainer.<init> New CoreContainer 1900833246
   [junit4]   2> 636145 T3416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/]
   [junit4]   2> 636145 T3416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 636146 T3416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 636146 T3416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 636146 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 636146 T3416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 636146 T3447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 636146 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 636147 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 636147 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 636147 T3416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 636148 T3416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 636148 T3416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 636148 T3416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 636149 T3416 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 636149 T3416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41552/solr
   [junit4]   2> 636149 T3416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 636150 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 636151 T3458 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@477c3b50 name:ZooKeeperConnection Watcher:127.0.0.1:41552 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 636152 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 636153 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 636154 T3460 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@55a0a496 name:ZooKeeperConnection Watcher:127.0.0.1:41552/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 636154 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 636167 T3416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 637169 T3416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42673_h_j%2Fd
   [junit4]   2> 637170 T3416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42673_h_j%2Fd
   [junit4]   2> 637172 T3438 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 637172 T3460 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 637172 T3447 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 637175 T3461 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 637175 T3461 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 637175 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 637175 T3461 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 637176 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 637176 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 637176 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 637177 T3439 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42673/h_j/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42673_h_j%2Fd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 637177 T3439 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 637177 T3439 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 637178 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 637280 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 637280 T3460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 637280 T3447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 638176 T3461 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 638176 T3461 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 638177 T3461 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 638177 T3461 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 638178 T3461 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 638178 T3461 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/collection1/'
   [junit4]   2> 638178 T3461 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 638179 T3461 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 638179 T3461 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 638197 T3461 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 638206 T3461 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 638207 T3461 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 638211 T3461 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 638312 T3461 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 638312 T3461 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 638312 T3461 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 638316 T3461 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 638317 T3461 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 638324 T3461 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 638326 T3461 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 638327 T3461 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 638328 T3461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 638328 T3461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 638328 T3461 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 638328 T3461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 638329 T3461 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 638329 T3461 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 638329 T3461 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 638329 T3461 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-003/collection1/, dataDir=./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty1/
   [junit4]   2> 638329 T3461 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@32e944e7
   [junit4]   2> 638330 T3461 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty1
   [junit4]   2> 638330 T3461 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty1/index/
   [junit4]   2> 638330 T3461 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 638330 T3461 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty1/index
   [junit4]   2> 638331 T3461 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 638331 T3461 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1da58236 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3aea0ca5),segFN=segments_1,generation=1}
   [junit4]   2> 638331 T3461 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 638333 T3461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 638334 T3461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 638334 T3461 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 638334 T3461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 638334 T3461 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 638334 T3461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 638335 T3461 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 638335 T3461 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 638335 T3461 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 638336 T3461 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 638336 T3461 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 638336 T3461 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 638337 T3461 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 638337 T3461 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 638337 T3461 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 638338 T3461 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 638339 T3461 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 638340 T3461 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 638340 T3461 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 638341 T3461 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=75.2890625, floorSegmentMB=1.884765625, forceMergeDeletesPctAllowed=16.58219904186481, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 638341 T3461 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1da58236 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3aea0ca5),segFN=segments_1,generation=1}
   [junit4]   2> 638341 T3461 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 638341 T3461 oass.SolrIndexSearcher.<init> Opening Searcher@6f081a7f[collection1] main
   [junit4]   2> 638342 T3461 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 638342 T3461 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 638343 T3461 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 638343 T3461 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 638343 T3461 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 638343 T3461 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 638344 T3461 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 638344 T3461 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 638344 T3461 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 638344 T3461 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 638346 T3462 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6f081a7f[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 638347 T3461 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 638348 T3465 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42673/h_j/d collection:collection1 shard:shard2
   [junit4]   2> 638348 T3416 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 638348 T3416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 638349 T3465 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 638352 T3465 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 638353 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638353 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638353 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638353 T3465 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 638353 T3465 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 638353 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C1058 name=collection1 org.apache.solr.core.SolrCore@3a34f5ef url=http://127.0.0.1:42673/h_j/d/collection1 node=127.0.0.1:42673_h_j%2Fd C1058_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:42673/h_j/d, core=collection1, node_name=127.0.0.1:42673_h_j%2Fd}
   [junit4]   2> 638353 T3465 C1058 P42673 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42673/h_j/d/collection1/
   [junit4]   2> 638354 T3465 C1058 P42673 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 638354 T3465 C1058 P42673 oasc.SyncStrategy.syncToMe http://127.0.0.1:42673/h_j/d/collection1/ has no replicas
   [junit4]   2> 638354 T3465 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42673/h_j/d/collection1/ shard2
   [junit4]   2> 638354 T3465 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 638355 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638358 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638359 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638462 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 638462 T3447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 638462 T3460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 638509 T3465 oasc.ZkController.register We are http://127.0.0.1:42673/h_j/d/collection1/ and leader is http://127.0.0.1:42673/h_j/d/collection1/
   [junit4]   2> 638509 T3465 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42673/h_j/d
   [junit4]   2> 638509 T3465 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 638509 T3465 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 638509 T3465 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 638510 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638511 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638511 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638511 T3465 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 638511 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 638512 T3439 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42673/h_j/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42673_h_j%2Fd",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 638513 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 638580 T3416 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 638580 T3416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 638582 T3416 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:45700
   [junit4]   2> 638583 T3416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 638583 T3416 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 638584 T3416 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004
   [junit4]   2> 638584 T3416 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/'
   [junit4]   2> 638596 T3416 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/solr.xml
   [junit4]   2> 638603 T3416 oasc.CoreContainer.<init> New CoreContainer 880105177
   [junit4]   2> 638603 T3416 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/]
   [junit4]   2> 638604 T3416 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 638604 T3416 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 638604 T3416 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 638605 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 638605 T3416 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 638605 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 638605 T3416 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 638606 T3416 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 638606 T3416 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 638606 T3416 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 638606 T3416 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 638607 T3416 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 638607 T3416 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41552/solr
   [junit4]   2> 638607 T3416 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 638608 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 638609 T3476 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@287a605d name:ZooKeeperConnection Watcher:127.0.0.1:41552 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 638610 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 638611 T3416 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 638612 T3478 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@dd41417 name:ZooKeeperConnection Watcher:127.0.0.1:41552/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 638612 T3416 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 638614 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 638614 T3460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 638614 T3447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 638620 T3416 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 639622 T3416 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45700_h_j%2Fd
   [junit4]   2> 639623 T3416 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45700_h_j%2Fd
   [junit4]   2> 639625 T3438 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 639625 T3447 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 639625 T3460 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 639625 T3478 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 639629 T3479 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 639629 T3479 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 639630 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 639630 T3479 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 639630 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 639630 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 639630 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 639631 T3439 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45700/h_j/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45700_h_j%2Fd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 639631 T3439 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 639631 T3439 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 639632 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 639734 T3478 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 639734 T3460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 639734 T3447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 639734 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 640630 T3479 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 640630 T3479 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 640631 T3479 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 640632 T3479 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 640633 T3479 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 640634 T3479 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/collection1/'
   [junit4]   2> 640635 T3479 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 640635 T3479 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 640635 T3479 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 640670 T3479 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 640685 T3479 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 640687 T3479 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 640704 T3479 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 640800 T3479 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 640800 T3479 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 640801 T3479 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 640804 T3479 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 640805 T3479 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 640817 T3479 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 640819 T3479 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 640820 T3479 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 640821 T3479 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 640821 T3479 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 640821 T3479 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 640823 T3479 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 640823 T3479 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 640823 T3479 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 640824 T3479 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 640824 T3479 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-004/collection1/, dataDir=./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty2/
   [junit4]   2> 640825 T3479 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@32e944e7
   [junit4]   2> 640826 T3479 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty2
   [junit4]   2> 640827 T3479 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty2/index/
   [junit4]   2> 640827 T3479 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 640827 T3479 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-001/jetty2/index
   [junit4]   2> 640828 T3479 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=44, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 640829 T3479 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4feddf19 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@534694e8),segFN=segments_1,generation=1}
   [junit4]   2> 640829 T3479 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 640831 T3479 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 640832 T3479 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 640832 T3479 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 640832 T3479 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 640832 T3479 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 640832 T3479 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 640833 T3479 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 640833 T3479 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 640833 T3479 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 640833 T3479 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 640834 T3479 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 640834 T3479 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 640834 T3479 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 640835 T3479 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 640835 T3479 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 640835 T3479 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 640837 T3479 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 640838 T3479 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 640838 T3479 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 640838 T3479 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=41, maxMergedSegmentMB=75.2890625, floorSegmentMB=1.884765625, forceMergeDeletesPctAllowed=16.58219904186481, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 640839 T3479 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4feddf19 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@534694e8),segFN=segments_1,generation=1}
   [junit4]   2> 640839 T3479 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 640839 T3479 oass.SolrIndexSearcher.<init> Opening Searcher@7e0481b9[collection1] main
   [junit4]   2> 640839 T3479 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 640840 T3479 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 640840 T3479 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 640841 T3479 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 640841 T3479 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 640841 T3479 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 640842 T3479 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 640842 T3479 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 640842 T3479 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 640842 T3479 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 640844 T3480 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7e0481b9[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 640847 T3479 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 640848 T3483 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:45700/h_j/d collection:collection1 shard:shard1
   [junit4]   2> 640848 T3416 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 640849 T3416 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 640849 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 640855 T3483 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 640856 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 640857 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 640857 T3483 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 640857 T3483 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1059 name=collection1 org.apache.solr.core.SolrCore@68ceb6ab url=http://127.0.0.1:45700/h_j/d/collection1 node=127.0.0.1:45700_h_j%2Fd C1059_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:45700/h_j/d, core=collection1, node_name=127.0.0.1:45700_h_j%2Fd}
   [junit4]   2> 640857 T3483 C1059 P45700 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:45700/h_j/d/collection1/
   [junit4]   2> 640858 T3483 C1059 P45700 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 640857 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 640858 T3483 C1059 P45700 oasc.SyncStrategy.syncToMe http://127.0.0.1:45700/h_j/d/collection1/ has no replicas
   [junit4]   2> 640859 T3483 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:45700/h_j/d/collection1/ shard1
   [junit4]   2> 640859 T3483 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 640860 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 640863 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 640865 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 640867 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 640968 T3438 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 640968 T3460 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 640968 T3447 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 640968 T3478 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 641016 T3483 oasc.ZkController.register We are http://127.0.0.1:45700/h_j/d/collection1/ and leader is http://127.0.0.1:45700/h_j/d/collection1/
   [junit4]   2> 641016 T3483 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:45700/h_j/d
   [junit4]   2> 641016 T3483 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 641016 T3483 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 641016 T3483 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 641018 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 641018 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 641018 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 641018 T3483 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 641019 T3439 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 641019 T3439 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:45700/h_j/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45700_h_j%2Fd",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 641021 T3438 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 641087 T3416 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 641087 T3416 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 641089 T3416 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:42798
   [junit4]   2> 641091 T3416 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 641091 T3416 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 641092 T3416 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.BasicDistributedZkTest-59A9E18F6772250E-001/tempDir-005
   [junit4]   

[...truncated too long message...]

ation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 762024 T4091 C1600 P36380 oasc.SyncStrategy.sync WARN Closed, skipping sync up.
   [junit4]   2> 762024 T4091 oasc.ShardLeaderElectionContext.rejoinLeaderElection Not rejoining election because CoreContainer is shutdown
   [junit4]   2> 762025 T4091 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@39b7cbb5
   [junit4]   2> 762026 T4091 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=3,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=172,cumulative_deletesById=74,cumulative_deletesByQuery=1,cumulative_errors=0,transaction_logs_total_size=14172,transaction_logs_total_number=1}
   [junit4]   2> 762027 T4091 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 762027 T4091 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 762027 T4091 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 762028 T4091 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 762029 T4091 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 762030 T4091 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.ChaosMonkeySafeLeaderTest-59A9E18F6772250E-001/tempDir-001/jetty10/index.20140523032500755 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.ChaosMonkeySafeLeaderTest-59A9E18F6772250E-001/tempDir-001/jetty10/index.20140523032500755;done=false>>]
   [junit4]   2> 762030 T4091 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.ChaosMonkeySafeLeaderTest-59A9E18F6772250E-001/tempDir-001/jetty10/index.20140523032500755
   [junit4]   2> 762031 T4091 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.ChaosMonkeySafeLeaderTest-59A9E18F6772250E-001/tempDir-001/jetty10 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.ChaosMonkeySafeLeaderTest-59A9E18F6772250E-001/tempDir-001/jetty10;done=false>>]
   [junit4]   2> 762031 T4091 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.ChaosMonkeySafeLeaderTest-59A9E18F6772250E-001/tempDir-001/jetty10
   [junit4]   2> 762031 T4091 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 762032 T4091 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 762032 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762032 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762032 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762033 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762033 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762033 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762033 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762034 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762034 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762034 T4091 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 762034 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 762035 T4091 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.ChaosMonkeySafeLeaderTest-59A9E18F6772250E-001
   [junit4]   2> NOTE: test params are: codec=Lucene42, sim=DefaultSimilarity, locale=bg_BG, timezone=SystemV/AST4
   [junit4]   2> NOTE: Linux 3.8.0-39-generic amd64/Oracle Corporation 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=166210088,total=527433728
   [junit4]   2> NOTE: All tests run in this JVM: [TestElisionMultitermQuery, TestRemoteStreaming, TestSolrIndexConfig, ReturnFieldsTest, SliceStateTest, BlockDirectoryTest, URLClassifyProcessorTest, TestManagedResourceStorage, CircularListTest, ParsingFieldUpdateProcessorsTest, JsonLoaderTest, TestMergePolicyConfig, MBeansHandlerTest, DateMathParserTest, CoreMergeIndexesAdminHandlerTest, CustomCollectionTest, SpatialFilterTest, LukeRequestHandlerTest, TestCollationFieldDocValues, TermVectorComponentTest, ClusterStateTest, FieldMutatingUpdateProcessorTest, TestCollationKeyRangeQueries, TestRandomDVFaceting, StressHdfsTest, UpdateRequestProcessorFactoryTest, TestSolrQueryParser, SOLR749Test, SynonymTokenizerTest, TestSuggestSpellingConverter, TestRandomMergePolicy, TestRestManager, TestLMJelinekMercerSimilarityFactory, HdfsBasicDistributedZkTest, ResponseLogComponentTest, TestSolrDeletionPolicy1, ScriptEngineTest, SoftAutoCommitTest, TestMiniSolrCloudCluster, TestDocumentBuilder, CopyFieldTest, AddBlockUpdateTest, OverseerCollectionProcessorTest, FastVectorHighlighterTest, SuggestComponentTest, TestExtendedDismaxParser, ClusterStateUpdateTest, TestSchemaSimilarityResource, ShardRoutingTest, TestJoin, TestSurroundQueryParser, TestNRTOpen, AsyncMigrateRouteKeyTest, TestStandardQParsers, DistribCursorPagingTest, HdfsBasicDistributedZk2Test, TestSolrXmlPersistor, TestFunctionQuery, TestDynamicFieldCollectionResource, CoreAdminCreateDiscoverTest, TestPostingsSolrHighlighter, ZkCLITest, ResourceLoaderTest, TestDocSet, SuggesterTest, TestSort, RegexBoostProcessorTest, BlockCacheTest, TestConfigSets, TestStressVersions, DistributedTermsComponentTest, QueryElevationComponentTest, AddSchemaFieldsUpdateProcessorFactoryTest, SpellPossibilityIteratorTest, BinaryUpdateRequestHandlerTest, MoreLikeThisHandlerTest, ShowFileRequestHandlerTest, TestCloudManagedSchema, TestLMDirichletSimilarityFactory, BasicFunctionalityTest, TestReversedWildcardFilterFactory, AutoCommitTest, TestTrie, TriLevelCompositeIdRoutingTest, WordBreakSolrSpellCheckerTest, TestLRUCache, TestCopyFieldCollectionResource, TestCloudManagedSchemaAddField, UpdateParamsTest, TestBlendedInfixSuggestions, TestManagedSchemaFieldResource, TestValueSourceCache, TestSearcherReuse, TestGroupingSearch, RankQueryTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J0 in 70.50s, 1 test, 1 failure <<< FAILURES!

[...truncated 618 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921: There were test failures: 396 suites, 1640 tests, 2 failures, 36 ignored (15 assumptions)

Total time: 81 minutes 10 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_20-ea-b11) - Build # 10246 - Still Failing!

Posted by Anshum Gupta <an...@anshumgupta.net>.
committed a fix.

On Fri, May 23, 2014 at 7:31 AM, Shalin Shekhar Mangar
<sh...@apache.org> wrote:
> This is related to SOLR-5681
>
>
> On Fri, May 23, 2014 at 7:57 PM, Policeman Jenkins Server
> <je...@thetaphi.de> wrote:
>>
>> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10246/
>> Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops -XX:+UseParallelGC
>>
>> 1 tests failed.
>> REGRESSION:
>> org.apache.solr.cloud.OverseerCollectionProcessorTest.testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>>
>> Error Message:
>> Captured an uncaught exception in thread: Thread[id=5060,
>> name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE,
>> group=TGRP-OverseerCollectionProcessorTest]
>>
>> Stack Trace:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=5060,
>> name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE,
>> group=TGRP-OverseerCollectionProcessorTest]
>>         at
>> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729:FD48244C03DACFD]:0)
>> Caused by: java.util.ConcurrentModificationException
>>         at __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
>>         at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
>>         at java.util.HashMap$KeyIterator.next(HashMap.java:1453)
>>         at
>> java.util.AbstractCollection.toString(AbstractCollection.java:461)
>>         at
>> org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
>>         at
>> org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
>>         at
>> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
>>         at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>         at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>         at java.lang.Thread.run(Thread.java:745)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 11410 lines...]
>>    [junit4] Suite: org.apache.solr.cloud.OverseerCollectionProcessorTest
>>    [junit4]   2> Creating dataDir:
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerCollectionProcessorTest-39FCE1E3ACCF5729-001/init-core-data-001
>>    [junit4]   2> 1092908 T5046 oas.SolrTestCaseJ4.buildSSLConfig
>> Randomized ssl (true) and clientAuth (true)
>>    [junit4]   2> 1093155 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>>    [junit4]   2> 1093178 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1093179 T5047 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1093197 T5047 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"6",
>>    [junit4]   2>          "maxShardsPerNode":"1"}
>>    [junit4]   2> 1093198 T5048
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"6",
>>    [junit4]   2>          "maxShardsPerNode":"1"}
>>    [junit4]   2> 1093198 T5048
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1093199 T5048 oasc.SolrException.log ERROR Collection
>> createcollection of createcollection
>> failed:org.apache.solr.common.SolrException: Cannot create collection
>> mycollection. Value of maxShardsPerNode is 1, and the number of live nodes
>> is 4. This allows a maximum of 4 to be created. Value of numShards is 6 and
>> value of replicationFactor is 1. This requires 6 shards to be created
>> (higher than the allowed number)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2>
>>    [junit4]   2> 1093205 T5048 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete, response:{Operation
>> createcollection caused exception:=org.apache.solr.common.SolrException:
>> Cannot create collection mycollection. Value of maxShardsPerNode is 1, and
>> the number of live nodes is 4. This allows a maximum of 4 to be created.
>> Value of numShards is 6 and value of replicationFactor is 1. This requires 6
>> shards to be created (higher than the allowed number),exception={msg=Cannot
>> create collection mycollection. Value of maxShardsPerNode is 1, and the
>> number of live nodes is 4. This allows a maximum of 4 to be created. Value
>> of numShards is 6 and value of replicationFactor is 1. This requires 6
>> shards to be created (higher than the allowed number),rspCode=400}}
>>    [junit4]   2> 1093282 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>>    [junit4]   2> 1093285 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testNoReplicationEqualNumberOfSlicesPerNode
>>    [junit4]   2> 1093290 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1093291 T5049 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1093291 T5049 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1093292 T5050
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1093292 T5050
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1093292 T5050
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1093393 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5,
>> shard6, shard7, shard8] , replicationFactor : 1
>>    [junit4]   2> 1093393 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1093395 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1093396 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1093396 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1093396 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard5_replica1 as part of slice shard5 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1093397 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard6_replica1 as part of slice shard6 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1093397 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard7_replica1 as part of slice shard7 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1093397 T5050
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard8_replica1 as part of slice shard8 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1093397 T5050
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1093398 T5050 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1093492 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testNoReplicationEqualNumberOfSlicesPerNode
>>    [junit4]   2> 1093495 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>>    [junit4]   2> 1093496 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1093496 T5051 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1093497 T5051 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>>    [junit4]   2> 1093497 T5052
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>>    [junit4]   2> 1093498 T5052
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1093498 T5052 oasc.SolrException.log ERROR Collection
>> createcollection of createcollection
>> failed:org.apache.solr.common.SolrException: Cannot create collection
>> mycollection. Value of maxShardsPerNode is 2, and the number of live nodes
>> is 3. This allows a maximum of 6 to be created. Value of numShards is 4 and
>> value of replicationFactor is 2. This requires 8 shards to be created
>> (higher than the allowed number)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2>
>>    [junit4]   2> 1093500 T5052 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete, response:{Operation
>> createcollection caused exception:=org.apache.solr.common.SolrException:
>> Cannot create collection mycollection. Value of maxShardsPerNode is 2, and
>> the number of live nodes is 3. This allows a maximum of 6 to be created.
>> Value of numShards is 4 and value of replicationFactor is 2. This requires 8
>> shards to be created (higher than the allowed number),exception={msg=Cannot
>> create collection mycollection. Value of maxShardsPerNode is 2, and the
>> number of live nodes is 3. This allows a maximum of 6 to be created. Value
>> of numShards is 4 and value of replicationFactor is 2. This requires 8
>> shards to be created (higher than the allowed number),rspCode=400}}
>>    [junit4]   2> 1093597 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>>    [junit4]   2> 1093601 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testNoReplicationLimitedNodesToCreateOn
>>    [junit4]   2> 1093603 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1093603 T5053 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1093603 T5053 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"6",
>>    [junit4]   2>          "maxShardsPerNode":"3",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>>    [junit4]   2> 1093604 T5054
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"6",
>>    [junit4]   2>          "maxShardsPerNode":"3",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>>    [junit4]   2> 1093604 T5054
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1093605 T5054
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1093706 T5054
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5,
>> shard6] , replicationFactor : 1
>>    [junit4]   2> 1093706 T5054
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1093707 T5054
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1093707 T5054
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1093707 T5054
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1093708 T5054
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard5_replica1 as part of slice shard5 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1093708 T5054
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard6_replica1 as part of slice shard6 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1093708 T5054
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1093709 T5054 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1093804 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testNoReplicationLimitedNodesToCreateOn
>>    [junit4]   2> 1093807 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>>    [junit4]   2> 1093808 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1093813 T5055 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1093815 T5055 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>>    [junit4]   2> 1093816 T5056
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>>    [junit4]   2> 1093816 T5056
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1093817 T5056
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1093917 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4] ,
>> replicationFactor : 2
>>    [junit4]   2> 1093917 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1093918 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica2 as part of slice shard1 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1093918 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1093918 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica2 as part of slice shard2 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1093918 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1093918 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica2 as part of slice shard3 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1093918 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1093919 T5056
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica2 as part of slice shard4 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1093919 T5056
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1093920 T5056 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1094014 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>>    [junit4]   2> 1094018 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testNoReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>>    [junit4]   2> 1094020 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1094020 T5057 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1094020 T5057 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>>    [junit4]   2> 1094021 T5058
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>>    [junit4]   2> 1094021 T5058
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1094021 T5058
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1094122 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5,
>> shard6, shard7, shard8] , replicationFactor : 1
>>    [junit4]   2> 1094122 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1094122 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094122 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094123 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1094123 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard5_replica1 as part of slice shard5 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1094123 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard6_replica1 as part of slice shard6 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094123 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard7_replica1 as part of slice shard7 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094124 T5058
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard8_replica1 as part of slice shard8 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1094124 T5058
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1094125 T5058 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1094221 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testNoReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>>    [junit4]   2> 1094226 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>>    [junit4]   2> 1094227 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1094228 T5059 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1094228 T5059 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>          "createNodeSet":null}
>>    [junit4]   2> 1094228 T5060
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>          "createNodeSet":null}
>>    [junit4]   2> 1094229 T5060
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1094229 T5060
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1094329 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4] ,
>> replicationFactor : 2
>>    [junit4]   2> 1094329 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1094330 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica2 as part of slice shard1 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094330 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094330 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica2 as part of slice shard2 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1094330 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1094331 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica2 as part of slice shard3 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094331 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094331 T5060
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica2 as part of slice shard4 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1094331 T5060
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1094333 T5060
>> oasc.OverseerCollectionProcessor$Runner.resetTaskWithException WARN
>> Resetting task: id, requestid: null, collectionName: mycollection
>>    [junit4]   2> 1426 T5060
>> ccr.RandomizedRunner$QueueUncaughtExceptionsHandler.uncaughtException
>> WARNING Uncaught exception in thread:
>> Thread[OverseerThreadFactory-2916-thread-1,5,TGRP-OverseerCollectionProcessorTest]
>> java.util.ConcurrentModificationException
>>    [junit4]   2>        at
>> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
>>    [junit4]   2>        at
>> java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
>>    [junit4]   2>        at
>> java.util.HashMap$KeyIterator.next(HashMap.java:1453)
>>    [junit4]   2>        at
>> java.util.AbstractCollection.toString(AbstractCollection.java:461)
>>    [junit4]   2>        at
>> org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
>>    [junit4]   2>        at
>> org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
>>    [junit4]   2>        at
>> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
>>    [junit4]   2>        at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>    [junit4]   2>        at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2>
>>    [junit4]   2> 1094428 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>>    [junit4]   2> NOTE: reproduce with: ant test
>> -Dtestcase=OverseerCollectionProcessorTest
>> -Dtests.method=testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>> -Dtests.seed=39FCE1E3ACCF5729 -Dtests.multiplier=3 -Dtests.slow=true
>> -Dtests.locale=sr_CS -Dtests.timezone=Asia/Ust-Nera
>> -Dtests.file.encoding=ISO-8859-1
>>    [junit4] ERROR   0.21s J1 |
>> OverseerCollectionProcessorTest.testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>> <<<
>>    [junit4]    > Throwable #1:
>> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
>> uncaught exception in thread: Thread[id=5060,
>> name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE,
>> group=TGRP-OverseerCollectionProcessorTest]
>>    [junit4]    >        at
>> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729:FD48244C03DACFD]:0)
>>    [junit4]    > Caused by: java.util.ConcurrentModificationException
>>    [junit4]    >        at
>> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
>>    [junit4]    >        at
>> java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
>>    [junit4]    >        at
>> java.util.HashMap$KeyIterator.next(HashMap.java:1453)
>>    [junit4]    >        at
>> java.util.AbstractCollection.toString(AbstractCollection.java:461)
>>    [junit4]    >        at
>> org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
>>    [junit4]    >        at
>> org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
>>    [junit4]    >        at
>> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
>>    [junit4]    >        at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>    [junit4]    >        at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2> 1094436 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>>    [junit4]   2> 1094436 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1094437 T5062 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1094438 T5062 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"3",
>>    [junit4]   2>          "maxShardsPerNode":"1"}
>>    [junit4]   2> 1094438 T5063
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"3",
>>    [junit4]   2>          "maxShardsPerNode":"1"}
>>    [junit4]   2> 1094438 T5063
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1094439 T5063 oasc.SolrException.log ERROR Collection
>> createcollection of createcollection
>> failed:org.apache.solr.common.SolrException: Cannot create collection
>> mycollection. Value of maxShardsPerNode is 1, and the number of live nodes
>> is 4. This allows a maximum of 4 to be created. Value of numShards is 3 and
>> value of replicationFactor is 2. This requires 6 shards to be created
>> (higher than the allowed number)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2>
>>    [junit4]   2> 1094442 T5063 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete, response:{Operation
>> createcollection caused exception:=org.apache.solr.common.SolrException:
>> Cannot create collection mycollection. Value of maxShardsPerNode is 1, and
>> the number of live nodes is 4. This allows a maximum of 4 to be created.
>> Value of numShards is 3 and value of replicationFactor is 2. This requires 6
>> shards to be created (higher than the allowed number),exception={msg=Cannot
>> create collection mycollection. Value of maxShardsPerNode is 1, and the
>> number of live nodes is 4. This allows a maximum of 4 to be created. Value
>> of numShards is 3 and value of replicationFactor is 2. This requires 6
>> shards to be created (higher than the allowed number),rspCode=400}}
>>    [junit4]   2> 1094537 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>>    [junit4]   2> 1094541 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testNoReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>>    [junit4]   2> 1094542 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1094542 T5064 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1094542 T5064 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>          "createNodeSet":null}
>>    [junit4]   2> 1094543 T5065
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>          "createNodeSet":null}
>>    [junit4]   2> 1094543 T5065
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1094543 T5065
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1094647 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5,
>> shard6, shard7, shard8] , replicationFactor : 1
>>    [junit4]   2> 1094647 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1094648 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1094648 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094648 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094648 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard5_replica1 as part of slice shard5 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1094649 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard6_replica1 as part of slice shard6 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1094649 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard7_replica1 as part of slice shard7 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094649 T5065
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard8_replica1 as part of slice shard8 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094649 T5065
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1094650 T5065 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1094743 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testNoReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>>    [junit4]   2> 1094747 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testReplicationUnequalNumberOfSlicesPerNode
>>    [junit4]   2> 1094748 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1094749 T5066 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1094749 T5066 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"3",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1094749 T5067
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"3",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1094750 T5067
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1094750 T5067
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1094850 T5067
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3] ,
>> replicationFactor : 2
>>    [junit4]   2> 1094851 T5067
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094851 T5067
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica2 as part of slice shard1 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094851 T5067
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1094851 T5067
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica2 as part of slice shard2 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1094851 T5067
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1094851 T5067
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica2 as part of slice shard3 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1094852 T5067
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1094853 T5067 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1094950 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testReplicationUnequalNumberOfSlicesPerNode
>>    [junit4]   2> 1094953 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testNoReplicationUnequalNumberOfSlicesPerNode
>>    [junit4]   2> 1094954 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1094955 T5068 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1094955 T5068 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"6",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1094955 T5069
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"6",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1094956 T5069
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1094956 T5069
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1095056 T5069
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5,
>> shard6] , replicationFactor : 1
>>    [junit4]   2> 1095056 T5069
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1095057 T5069
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1095057 T5069
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1095057 T5069
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1095058 T5069
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard5_replica1 as part of slice shard5 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1095058 T5069
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard6_replica1 as part of slice shard6 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1095059 T5069
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1095059 T5069 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1095155 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testNoReplicationUnequalNumberOfSlicesPerNode
>>    [junit4]   2> 1095159 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testReplicationEqualNumberOfSlicesPerNode
>>    [junit4]   2> 1095160 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1095160 T5070 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1095160 T5070 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1095161 T5071
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"4",
>>    [junit4]   2>          "maxShardsPerNode":"2"}
>>    [junit4]   2> 1095161 T5071
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1095161 T5071
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1095262 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3, shard4] ,
>> replicationFactor : 2
>>    [junit4]   2> 1095262 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1095262 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica2 as part of slice shard1 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1095263 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1095263 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica2 as part of slice shard2 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1095263 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8966_solr
>>    [junit4]   2> 1095263 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica2 as part of slice shard3 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1095264 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica1 as part of slice shard4 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1095264 T5071
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard4_replica2 as part of slice shard4 of collection
>> mycollection on localhost:8965_solr
>>    [junit4]   2> 1095264 T5071
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1095265 T5071 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1095361 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testReplicationEqualNumberOfSlicesPerNode
>>    [junit4]   2> 1095364 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>>    [junit4]   2> 1095365 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1095365 T5072 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1095366 T5072 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>>    [junit4]   2> 1095366 T5073
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"1",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"8",
>>    [junit4]   2>          "maxShardsPerNode":"2",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>>    [junit4]   2> 1095366 T5073
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1095367 T5073 oasc.SolrException.log ERROR Collection
>> createcollection of createcollection
>> failed:org.apache.solr.common.SolrException: Cannot create collection
>> mycollection. Value of maxShardsPerNode is 2, and the number of live nodes
>> is 3. This allows a maximum of 6 to be created. Value of numShards is 8 and
>> value of replicationFactor is 1. This requires 8 shards to be created
>> (higher than the allowed number)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>>    [junit4]   2>                at
>> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>>    [junit4]   2>                at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2>
>>    [junit4]   2> 1095368 T5073 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete, response:{Operation
>> createcollection caused exception:=org.apache.solr.common.SolrException:
>> Cannot create collection mycollection. Value of maxShardsPerNode is 2, and
>> the number of live nodes is 3. This allows a maximum of 6 to be created.
>> Value of numShards is 8 and value of replicationFactor is 1. This requires 8
>> shards to be created (higher than the allowed number),exception={msg=Cannot
>> create collection mycollection. Value of maxShardsPerNode is 2, and the
>> number of live nodes is 3. This allows a maximum of 6 to be created. Value
>> of numShards is 8 and value of replicationFactor is 1. This requires 8
>> shards to be created (higher than the allowed number),rspCode=400}}
>>    [junit4]   2> 1095466 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>>    [junit4]   2> 1095469 T5046 oas.SolrTestCaseJ4.setUp ###Starting
>> testReplicationLimitedNodesToCreateOn
>>    [junit4]   2> 1095470 T5046
>> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>>    [junit4]   2> 1095470 T5074 oasc.OverseerCollectionProcessor.run
>> Process current queue of collection creations
>>    [junit4]   2> 1095471 T5074 oasc.OverseerCollectionProcessor.run
>> Overseer Collection Processor: Get the message id:id message:{
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"3",
>>    [junit4]   2>          "maxShardsPerNode":"3",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>>    [junit4]   2> 1095471 T5075
>> oasc.OverseerCollectionProcessor.processMessage WARN
>> OverseerCollectionProcessor.processMessage : createcollection , {
>>    [junit4]   2>          "operation":"createcollection",
>>    [junit4]   2>          "replicationFactor":"2",
>>    [junit4]   2>          "name":"mycollection",
>>    [junit4]   2>          "collection.configName":"myconfig",
>>    [junit4]   2>          "numShards":"3",
>>    [junit4]   2>          "maxShardsPerNode":"3",
>>    [junit4]   2>
>> "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>>    [junit4]   2> 1095471 T5075
>> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>>    [junit4]   2> 1095471 T5075
>> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
>> node /collections/mycollection
>>    [junit4]   2> 1095572 T5075
>> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new
>> collection mycollection, shardNames [shard1, shard2, shard3] ,
>> replicationFactor : 2
>>    [junit4]   2> 1095572 T5075
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica1 as part of slice shard1 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1095572 T5075
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard1_replica2 as part of slice shard1 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1095573 T5075
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica1 as part of slice shard2 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1095573 T5075
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard2_replica2 as part of slice shard2 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1095573 T5075
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica1 as part of slice shard3 of collection
>> mycollection on localhost:8964_solr
>>    [junit4]   2> 1095573 T5075
>> oasc.OverseerCollectionProcessor.createCollection Creating shard
>> mycollection_shard3_replica2 as part of slice shard3 of collection
>> mycollection on localhost:8963_solr
>>    [junit4]   2> 1095573 T5075
>> oasc.OverseerCollectionProcessor.createCollection Finished create command on
>> all shards for collection: mycollection
>>    [junit4]   2> 1095574 T5075 oasc.OverseerCollectionProcessor$Runner.run
>> Overseer Collection Processor: Message id:id complete,
>> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>>    [junit4]   2> 1095671 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
>> testReplicationLimitedNodesToCreateOn
>>    [junit4]   2> 1095673 T5046 oas.SolrTestCaseJ4.deleteCore ###deleteCore
>>    [junit4]   2> NOTE: test params are: codec=Lucene40,
>> sim=DefaultSimilarity, locale=sr_CS, timezone=Asia/Ust-Nera
>>    [junit4]   2> NOTE: Linux 3.8.0-39-generic amd64/Oracle Corporation
>> 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=155561560,total=386400256
>>    [junit4]   2> NOTE: All tests run in this JVM:
>> [DistributedExpandComponentTest, ConvertedLegacyTest, TestNumberUtils,
>> TestRecovery, StatsComponentTest, TestRandomFaceting,
>> AlternateDirectoryTest, TestSolrXml, TestClassNameShortening,
>> TestIndexingPerformance, TestSchemaResource, HighlighterConfigTest,
>> SignatureUpdateProcessorFactoryTest, StandardRequestHandlerTest,
>> TestDefaultSimilarityFactory, NumericFieldsTest, OverseerStatusTest,
>> DateFieldTest, TestExpandComponent, TestFastOutputStream,
>> TestIBSimilarityFactory, TestFreeTextSuggestions, OpenCloseCoreStressTest,
>> ResponseHeaderTest, SuggesterWFSTTest, RequestHandlersTest,
>> TestCloudInspectUtil, TestFileDictionaryLookup, HdfsSyncSliceTest,
>> SolrXmlInZkTest, IndexBasedSpellCheckerTest, DeleteReplicaTest,
>> NoCacheHeaderTest, TestCSVResponseWriter, CursorMarkTest, TestFastLRUCache,
>> CurrencyFieldXmlFileTest, TestIndexSearcher, ConnectionManagerTest,
>> ZkSolrClientTest, DisMaxRequestHandlerTest, AtomicUpdatesTest,
>> TestStressReorder, PrimUtilsTest, UniqFieldsUpdateProcessorFactoryTest,
>> TestSolrJ, TestBinaryResponseWriter, DirectSolrSpellCheckerTest,
>> RAMDirectoryFactoryTest, NotRequiredUniqueKeyTest,
>> CoreAdminRequestStatusTest, BadCopyFieldTest,
>> HdfsWriteToMultipleCollectionsTest, TestPartialUpdateDeduplication,
>> TestDocBasedVersionConstraints, CoreContainerCoreInitFailuresTest,
>> OutputWriterTest, JSONWriterTest, TestInfoStreamLogging, SimplePostToolTest,
>> PingRequestHandlerTest, TestHighFrequencyDictionaryFactory,
>> TestAnalyzedSuggestions, TestFieldCollectionResource, RemoteQueryErrorTest,
>> ChaosMonkeySafeLeaderTest, CurrencyFieldOpenExchangeTest,
>> TestArbitraryIndexDir, TestInitQParser, TestSolrQueryParserResource,
>> TestNoOpRegenerator, CollectionsAPIDistributedZkTest, TestCoreDiscovery,
>> TestSolrCoreProperties, HardAutoCommitTest,
>> HdfsCollectionsAPIDistributedZkTest, TestCustomSort,
>> DeleteInactiveReplicaTest, DistributedQueryComponentOptimizationTest,
>> TestSimpleQParserPlugin, TestShardHandlerFactory, SolrIndexConfigTest,
>> TestStressRecovery, OpenExchangeRatesOrgProviderTest,
>> SpellCheckComponentTest, ChaosMonkeyNothingIsSafeTest, TestQueryTypes,
>> DirectSolrConnectionTest, TestLazyCores, TestHighlightDedupGrouping,
>> TestCollectionAPI, MultiTermTest, SpellCheckCollatorTest, TestUtils,
>> BasicDistributedZk2Test, LeaderElectionIntegrationTest, BasicZkTest,
>> TestReplicationHandler, LeaderElectionTest, ShardRoutingCustomTest,
>> SliceStateUpdateTest, TestDistributedSearch, TestDistributedGrouping,
>> TestHashPartitioner, TermVectorComponentDistributedTest, TestJoin,
>> TestCoreContainer, TestSolr4Spatial, TestGroupingSearch, TestFunctionQuery,
>> DirectUpdateHandlerTest, OverseerCollectionProcessorTest]
>>    [junit4] Completed on J1 in 2.78s, 14 tests, 1 error <<< FAILURES!
>>
>> [...truncated 498 lines...]
>> BUILD FAILED
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The
>> following error occurred while executing this line:
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The
>> following error occurred while executing this line:
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The
>> following error occurred while executing this line:
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The
>> following error occurred while executing this line:
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The
>> following error occurred while executing this line:
>>
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496:
>> The following error occurred while executing this line:
>>
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297:
>> The following error occurred while executing this line:
>>
>> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921:
>> There were test failures: 396 suites, 1640 tests, 1 error, 36 ignored (15
>> assumptions)
>>
>> Total time: 59 minutes 32 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Description set: Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops
>> -XX:+UseParallelGC
>> Archiving artifacts
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>>
>
>
>
> --
> Regards,
> Shalin Shekhar Mangar.



-- 

Anshum Gupta
http://www.anshumgupta.net

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_20-ea-b11) - Build # 10246 - Still Failing!

Posted by Shalin Shekhar Mangar <sh...@apache.org>.
This is related to SOLR-5681


On Fri, May 23, 2014 at 7:57 PM, Policeman Jenkins Server <
jenkins@thetaphi.de> wrote:

> Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10246/
> Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops -XX:+UseParallelGC
>
> 1 tests failed.
> REGRESSION:
>  org.apache.solr.cloud.OverseerCollectionProcessorTest.testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>
> Error Message:
> Captured an uncaught exception in thread: Thread[id=5060,
> name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE,
> group=TGRP-OverseerCollectionProcessorTest]
>
> Stack Trace:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=5060,
> name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE,
> group=TGRP-OverseerCollectionProcessorTest]
>         at
> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729:FD48244C03DACFD]:0)
> Caused by: java.util.ConcurrentModificationException
>         at __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
>         at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
>         at java.util.HashMap$KeyIterator.next(HashMap.java:1453)
>         at
> java.util.AbstractCollection.toString(AbstractCollection.java:461)
>         at
> org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
>         at
> org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
>         at
> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
>         at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>         at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 11410 lines...]
>    [junit4] Suite: org.apache.solr.cloud.OverseerCollectionProcessorTest
>    [junit4]   2> Creating dataDir:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerCollectionProcessorTest-39FCE1E3ACCF5729-001/init-core-data-001
>    [junit4]   2> 1092908 T5046 oas.SolrTestCaseJ4.buildSSLConfig
> Randomized ssl (true) and clientAuth (true)
>    [junit4]   2> 1093155 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>    [junit4]   2> 1093178 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1093179 T5047 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1093197 T5047 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"6",
>    [junit4]   2>          "maxShardsPerNode":"1"}
>    [junit4]   2> 1093198 T5048
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"6",
>    [junit4]   2>          "maxShardsPerNode":"1"}
>    [junit4]   2> 1093198 T5048
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1093199 T5048 oasc.SolrException.log ERROR Collection
> createcollection of createcollection
> failed:org.apache.solr.common.SolrException: Cannot create collection
> mycollection. Value of maxShardsPerNode is 1, and the number of live nodes
> is 4. This allows a maximum of 4 to be created. Value of numShards is 6 and
> value of replicationFactor is 1. This requires 6 shards to be created
> (higher than the allowed number)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 1093205 T5048 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete, response:{Operation
> createcollection caused exception:=org.apache.solr.common.SolrException:
> Cannot create collection mycollection. Value of maxShardsPerNode is 1, and
> the number of live nodes is 4. This allows a maximum of 4 to be created.
> Value of numShards is 6 and value of replicationFactor is 1. This requires
> 6 shards to be created (higher than the allowed
> number),exception={msg=Cannot create collection mycollection. Value of
> maxShardsPerNode is 1, and the number of live nodes is 4. This allows a
> maximum of 4 to be created. Value of numShards is 6 and value of
> replicationFactor is 1. This requires 6 shards to be created (higher than
> the allowed number),rspCode=400}}
>    [junit4]   2> 1093282 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>    [junit4]   2> 1093285 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testNoReplicationEqualNumberOfSlicesPerNode
>    [junit4]   2> 1093290 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1093291 T5049 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1093291 T5049 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1093292 T5050
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1093292 T5050
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1093292 T5050
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1093393 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4,
> shard5, shard6, shard7, shard8] , replicationFactor : 1
>    [junit4]   2> 1093393 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1093395 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1093396 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1093396 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1093396 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard5_replica1 as part of slice shard5 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1093397 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard6_replica1 as part of slice shard6 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1093397 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard7_replica1 as part of slice shard7 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1093397 T5050
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard8_replica1 as part of slice shard8 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1093397 T5050
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1093398 T5050 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1093492 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testNoReplicationEqualNumberOfSlicesPerNode
>    [junit4]   2> 1093495 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>    [junit4]   2> 1093496 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1093496 T5051 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1093497 T5051 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>    [junit4]   2> 1093497 T5052
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>    [junit4]   2> 1093498 T5052
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1093498 T5052 oasc.SolrException.log ERROR Collection
> createcollection of createcollection
> failed:org.apache.solr.common.SolrException: Cannot create collection
> mycollection. Value of maxShardsPerNode is 2, and the number of live nodes
> is 3. This allows a maximum of 6 to be created. Value of numShards is 4 and
> value of replicationFactor is 2. This requires 8 shards to be created
> (higher than the allowed number)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 1093500 T5052 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete, response:{Operation
> createcollection caused exception:=org.apache.solr.common.SolrException:
> Cannot create collection mycollection. Value of maxShardsPerNode is 2, and
> the number of live nodes is 3. This allows a maximum of 6 to be created.
> Value of numShards is 4 and value of replicationFactor is 2. This requires
> 8 shards to be created (higher than the allowed
> number),exception={msg=Cannot create collection mycollection. Value of
> maxShardsPerNode is 2, and the number of live nodes is 3. This allows a
> maximum of 6 to be created. Value of numShards is 4 and value of
> replicationFactor is 2. This requires 8 shards to be created (higher than
> the allowed number),rspCode=400}}
>    [junit4]   2> 1093597 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>    [junit4]   2> 1093601 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testNoReplicationLimitedNodesToCreateOn
>    [junit4]   2> 1093603 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1093603 T5053 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1093603 T5053 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"6",
>    [junit4]   2>          "maxShardsPerNode":"3",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>    [junit4]   2> 1093604 T5054
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"6",
>    [junit4]   2>          "maxShardsPerNode":"3",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>    [junit4]   2> 1093604 T5054
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1093605 T5054
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1093706 T5054
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4,
> shard5, shard6] , replicationFactor : 1
>    [junit4]   2> 1093706 T5054
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1093707 T5054
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1093707 T5054
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1093707 T5054
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1093708 T5054
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard5_replica1 as part of slice shard5 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1093708 T5054
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard6_replica1 as part of slice shard6 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1093708 T5054
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1093709 T5054 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1093804 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testNoReplicationLimitedNodesToCreateOn
>    [junit4]   2> 1093807 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>    [junit4]   2> 1093808 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1093813 T5055 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1093815 T5055 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>    [junit4]   2> 1093816 T5056
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>    [junit4]   2> 1093816 T5056
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1093817 T5056
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1093917 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4] ,
> replicationFactor : 2
>    [junit4]   2> 1093917 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1093918 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica2 as part of slice shard1 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1093918 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1093918 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica2 as part of slice shard2 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1093918 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1093918 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica2 as part of slice shard3 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1093918 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1093919 T5056
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica2 as part of slice shard4 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1093919 T5056
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1093920 T5056 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1094014 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>    [junit4]   2> 1094018 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testNoReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>    [junit4]   2> 1094020 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1094020 T5057 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1094020 T5057 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>    [junit4]   2> 1094021 T5058
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
>    [junit4]   2> 1094021 T5058
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1094021 T5058
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1094122 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4,
> shard5, shard6, shard7, shard8] , replicationFactor : 1
>    [junit4]   2> 1094122 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1094122 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094122 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094123 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1094123 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard5_replica1 as part of slice shard5 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1094123 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard6_replica1 as part of slice shard6 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094123 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard7_replica1 as part of slice shard7 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094124 T5058
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard8_replica1 as part of slice shard8 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1094124 T5058
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1094125 T5058 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1094221 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testNoReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
>    [junit4]   2> 1094226 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>    [junit4]   2> 1094227 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1094228 T5059 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1094228 T5059 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>          "createNodeSet":null}
>    [junit4]   2> 1094228 T5060
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>          "createNodeSet":null}
>    [junit4]   2> 1094229 T5060
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1094229 T5060
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1094329 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4] ,
> replicationFactor : 2
>    [junit4]   2> 1094329 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1094330 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica2 as part of slice shard1 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094330 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094330 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica2 as part of slice shard2 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1094330 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1094331 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica2 as part of slice shard3 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094331 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094331 T5060
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica2 as part of slice shard4 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1094331 T5060
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1094333 T5060
> oasc.OverseerCollectionProcessor$Runner.resetTaskWithException WARN
> Resetting task: id, requestid: null, collectionName: mycollection
>    [junit4]   2> 1426 T5060
> ccr.RandomizedRunner$QueueUncaughtExceptionsHandler.uncaughtException
> WARNING Uncaught exception in thread:
> Thread[OverseerThreadFactory-2916-thread-1,5,TGRP-OverseerCollectionProcessorTest]
> java.util.ConcurrentModificationException
>    [junit4]   2>        at
> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
>    [junit4]   2>        at
> java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
>    [junit4]   2>        at
> java.util.HashMap$KeyIterator.next(HashMap.java:1453)
>    [junit4]   2>        at
> java.util.AbstractCollection.toString(AbstractCollection.java:461)
>    [junit4]   2>        at
> org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
>    [junit4]   2>        at
> org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
>    [junit4]   2>        at
> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
>    [junit4]   2>        at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>    [junit4]   2>        at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>    [junit4]   2>        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 1094428 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>    [junit4]   2> NOTE: reproduce with: ant test
>  -Dtestcase=OverseerCollectionProcessorTest
> -Dtests.method=testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
> -Dtests.seed=39FCE1E3ACCF5729 -Dtests.multiplier=3 -Dtests.slow=true
> -Dtests.locale=sr_CS -Dtests.timezone=Asia/Ust-Nera
> -Dtests.file.encoding=ISO-8859-1
>    [junit4] ERROR   0.21s J1 |
> OverseerCollectionProcessorTest.testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
> <<<
>    [junit4]    > Throwable #1:
> com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an
> uncaught exception in thread: Thread[id=5060,
> name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE,
> group=TGRP-OverseerCollectionProcessorTest]
>    [junit4]    >        at
> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729:FD48244C03DACFD]:0)
>    [junit4]    > Caused by: java.util.ConcurrentModificationException
>    [junit4]    >        at
> __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
>    [junit4]    >        at
> java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
>    [junit4]    >        at
> java.util.HashMap$KeyIterator.next(HashMap.java:1453)
>    [junit4]    >        at
> java.util.AbstractCollection.toString(AbstractCollection.java:461)
>    [junit4]    >        at
> org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
>    [junit4]    >        at
> org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
>    [junit4]    >        at
> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
>    [junit4]    >        at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>    [junit4]    >        at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> 1094436 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>    [junit4]   2> 1094436 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1094437 T5062 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1094438 T5062 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "maxShardsPerNode":"1"}
>    [junit4]   2> 1094438 T5063
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "maxShardsPerNode":"1"}
>    [junit4]   2> 1094438 T5063
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1094439 T5063 oasc.SolrException.log ERROR Collection
> createcollection of createcollection
> failed:org.apache.solr.common.SolrException: Cannot create collection
> mycollection. Value of maxShardsPerNode is 1, and the number of live nodes
> is 4. This allows a maximum of 4 to be created. Value of numShards is 3 and
> value of replicationFactor is 2. This requires 6 shards to be created
> (higher than the allowed number)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 1094442 T5063 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete, response:{Operation
> createcollection caused exception:=org.apache.solr.common.SolrException:
> Cannot create collection mycollection. Value of maxShardsPerNode is 1, and
> the number of live nodes is 4. This allows a maximum of 4 to be created.
> Value of numShards is 3 and value of replicationFactor is 2. This requires
> 6 shards to be created (higher than the allowed
> number),exception={msg=Cannot create collection mycollection. Value of
> maxShardsPerNode is 1, and the number of live nodes is 4. This allows a
> maximum of 4 to be created. Value of numShards is 3 and value of
> replicationFactor is 2. This requires 6 shards to be created (higher than
> the allowed number),rspCode=400}}
>    [junit4]   2> 1094537 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
>    [junit4]   2> 1094541 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testNoReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>    [junit4]   2> 1094542 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1094542 T5064 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1094542 T5064 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>          "createNodeSet":null}
>    [junit4]   2> 1094543 T5065
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>          "createNodeSet":null}
>    [junit4]   2> 1094543 T5065
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1094543 T5065
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1094647 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4,
> shard5, shard6, shard7, shard8] , replicationFactor : 1
>    [junit4]   2> 1094647 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1094648 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1094648 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094648 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094648 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard5_replica1 as part of slice shard5 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1094649 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard6_replica1 as part of slice shard6 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1094649 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard7_replica1 as part of slice shard7 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094649 T5065
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard8_replica1 as part of slice shard8 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094649 T5065
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1094650 T5065 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1094743 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testNoReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
>    [junit4]   2> 1094747 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testReplicationUnequalNumberOfSlicesPerNode
>    [junit4]   2> 1094748 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1094749 T5066 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1094749 T5066 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1094749 T5067
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1094750 T5067
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1094750 T5067
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1094850 T5067
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3] ,
> replicationFactor : 2
>    [junit4]   2> 1094851 T5067
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094851 T5067
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica2 as part of slice shard1 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094851 T5067
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1094851 T5067
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica2 as part of slice shard2 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1094851 T5067
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1094851 T5067
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica2 as part of slice shard3 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1094852 T5067
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1094853 T5067 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1094950 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testReplicationUnequalNumberOfSlicesPerNode
>    [junit4]   2> 1094953 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testNoReplicationUnequalNumberOfSlicesPerNode
>    [junit4]   2> 1094954 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1094955 T5068 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1094955 T5068 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"6",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1094955 T5069
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"6",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1094956 T5069
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1094956 T5069
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1095056 T5069
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4,
> shard5, shard6] , replicationFactor : 1
>    [junit4]   2> 1095056 T5069
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1095057 T5069
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1095057 T5069
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1095057 T5069
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1095058 T5069
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard5_replica1 as part of slice shard5 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1095058 T5069
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard6_replica1 as part of slice shard6 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1095059 T5069
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1095059 T5069 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1095155 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testNoReplicationUnequalNumberOfSlicesPerNode
>    [junit4]   2> 1095159 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testReplicationEqualNumberOfSlicesPerNode
>    [junit4]   2> 1095160 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1095160 T5070 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1095160 T5070 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1095161 T5071
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"4",
>    [junit4]   2>          "maxShardsPerNode":"2"}
>    [junit4]   2> 1095161 T5071
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1095161 T5071
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1095262 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3, shard4] ,
> replicationFactor : 2
>    [junit4]   2> 1095262 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1095262 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica2 as part of slice shard1 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1095263 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1095263 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica2 as part of slice shard2 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1095263 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8966_solr
>    [junit4]   2> 1095263 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica2 as part of slice shard3 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1095264 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica1 as part of slice shard4 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1095264 T5071
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard4_replica2 as part of slice shard4 of collection
> mycollection on localhost:8965_solr
>    [junit4]   2> 1095264 T5071
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1095265 T5071 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1095361 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testReplicationEqualNumberOfSlicesPerNode
>    [junit4]   2> 1095364 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>    [junit4]   2> 1095365 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1095365 T5072 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1095366 T5072 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>    [junit4]   2> 1095366 T5073
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"1",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"8",
>    [junit4]   2>          "maxShardsPerNode":"2",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
>    [junit4]   2> 1095366 T5073
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1095367 T5073 oasc.SolrException.log ERROR Collection
> createcollection of createcollection
> failed:org.apache.solr.common.SolrException: Cannot create collection
> mycollection. Value of maxShardsPerNode is 2, and the number of live nodes
> is 3. This allows a maximum of 6 to be created. Value of numShards is 8 and
> value of replicationFactor is 1. This requires 8 shards to be created
> (higher than the allowed number)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
>    [junit4]   2>                at
> org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
>    [junit4]   2>                at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>    [junit4]   2>                at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2>
>    [junit4]   2> 1095368 T5073 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete, response:{Operation
> createcollection caused exception:=org.apache.solr.common.SolrException:
> Cannot create collection mycollection. Value of maxShardsPerNode is 2, and
> the number of live nodes is 3. This allows a maximum of 6 to be created.
> Value of numShards is 8 and value of replicationFactor is 1. This requires
> 8 shards to be created (higher than the allowed
> number),exception={msg=Cannot create collection mycollection. Value of
> maxShardsPerNode is 2, and the number of live nodes is 3. This allows a
> maximum of 6 to be created. Value of numShards is 8 and value of
> replicationFactor is 1. This requires 8 shards to be created (higher than
> the allowed number),rspCode=400}}
>    [junit4]   2> 1095466 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
>    [junit4]   2> 1095469 T5046 oas.SolrTestCaseJ4.setUp ###Starting
> testReplicationLimitedNodesToCreateOn
>    [junit4]   2> 1095470 T5046
> oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
>    [junit4]   2> 1095470 T5074 oasc.OverseerCollectionProcessor.run
> Process current queue of collection creations
>    [junit4]   2> 1095471 T5074 oasc.OverseerCollectionProcessor.run
> Overseer Collection Processor: Get the message id:id message:{
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "maxShardsPerNode":"3",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>    [junit4]   2> 1095471 T5075
> oasc.OverseerCollectionProcessor.processMessage WARN
> OverseerCollectionProcessor.processMessage : createcollection , {
>    [junit4]   2>          "operation":"createcollection",
>    [junit4]   2>          "replicationFactor":"2",
>    [junit4]   2>          "name":"mycollection",
>    [junit4]   2>          "collection.configName":"myconfig",
>    [junit4]   2>          "numShards":"3",
>    [junit4]   2>          "maxShardsPerNode":"3",
>    [junit4]   2>
>  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
>    [junit4]   2> 1095471 T5075
> oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
>    [junit4]   2> 1095471 T5075
> oasc.OverseerCollectionProcessor.createConfNode creating collections conf
> node /collections/mycollection
>    [junit4]   2> 1095572 T5075
> oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for
> new collection mycollection, shardNames [shard1, shard2, shard3] ,
> replicationFactor : 2
>    [junit4]   2> 1095572 T5075
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica1 as part of slice shard1 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1095572 T5075
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard1_replica2 as part of slice shard1 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1095573 T5075
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica1 as part of slice shard2 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1095573 T5075
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard2_replica2 as part of slice shard2 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1095573 T5075
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica1 as part of slice shard3 of collection
> mycollection on localhost:8964_solr
>    [junit4]   2> 1095573 T5075
> oasc.OverseerCollectionProcessor.createCollection Creating shard
> mycollection_shard3_replica2 as part of slice shard3 of collection
> mycollection on localhost:8963_solr
>    [junit4]   2> 1095573 T5075
> oasc.OverseerCollectionProcessor.createCollection Finished create command
> on all shards for collection: mycollection
>    [junit4]   2> 1095574 T5075 oasc.OverseerCollectionProcessor$Runner.run
> Overseer Collection Processor: Message id:id complete,
> response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
>    [junit4]   2> 1095671 T5046 oas.SolrTestCaseJ4.tearDown ###Ending
> testReplicationLimitedNodesToCreateOn
>    [junit4]   2> 1095673 T5046 oas.SolrTestCaseJ4.deleteCore ###deleteCore
>    [junit4]   2> NOTE: test params are: codec=Lucene40,
> sim=DefaultSimilarity, locale=sr_CS, timezone=Asia/Ust-Nera
>    [junit4]   2> NOTE: Linux 3.8.0-39-generic amd64/Oracle Corporation
> 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=155561560,total=386400256
>    [junit4]   2> NOTE: All tests run in this JVM:
> [DistributedExpandComponentTest, ConvertedLegacyTest, TestNumberUtils,
> TestRecovery, StatsComponentTest, TestRandomFaceting,
> AlternateDirectoryTest, TestSolrXml, TestClassNameShortening,
> TestIndexingPerformance, TestSchemaResource, HighlighterConfigTest,
> SignatureUpdateProcessorFactoryTest, StandardRequestHandlerTest,
> TestDefaultSimilarityFactory, NumericFieldsTest, OverseerStatusTest,
> DateFieldTest, TestExpandComponent, TestFastOutputStream,
> TestIBSimilarityFactory, TestFreeTextSuggestions, OpenCloseCoreStressTest,
> ResponseHeaderTest, SuggesterWFSTTest, RequestHandlersTest,
> TestCloudInspectUtil, TestFileDictionaryLookup, HdfsSyncSliceTest,
> SolrXmlInZkTest, IndexBasedSpellCheckerTest, DeleteReplicaTest,
> NoCacheHeaderTest, TestCSVResponseWriter, CursorMarkTest, TestFastLRUCache,
> CurrencyFieldXmlFileTest, TestIndexSearcher, ConnectionManagerTest,
> ZkSolrClientTest, DisMaxRequestHandlerTest, AtomicUpdatesTest,
> TestStressReorder, PrimUtilsTest, UniqFieldsUpdateProcessorFactoryTest,
> TestSolrJ, TestBinaryResponseWriter, DirectSolrSpellCheckerTest,
> RAMDirectoryFactoryTest, NotRequiredUniqueKeyTest,
> CoreAdminRequestStatusTest, BadCopyFieldTest,
> HdfsWriteToMultipleCollectionsTest, TestPartialUpdateDeduplication,
> TestDocBasedVersionConstraints, CoreContainerCoreInitFailuresTest,
> OutputWriterTest, JSONWriterTest, TestInfoStreamLogging,
> SimplePostToolTest, PingRequestHandlerTest,
> TestHighFrequencyDictionaryFactory, TestAnalyzedSuggestions,
> TestFieldCollectionResource, RemoteQueryErrorTest,
> ChaosMonkeySafeLeaderTest, CurrencyFieldOpenExchangeTest,
> TestArbitraryIndexDir, TestInitQParser, TestSolrQueryParserResource,
> TestNoOpRegenerator, CollectionsAPIDistributedZkTest, TestCoreDiscovery,
> TestSolrCoreProperties, HardAutoCommitTest,
> HdfsCollectionsAPIDistributedZkTest, TestCustomSort,
> DeleteInactiveReplicaTest, DistributedQueryComponentOptimizationTest,
> TestSimpleQParserPlugin, TestShardHandlerFactory, SolrIndexConfigTest,
> TestStressRecovery, OpenExchangeRatesOrgProviderTest,
> SpellCheckComponentTest, ChaosMonkeyNothingIsSafeTest, TestQueryTypes,
> DirectSolrConnectionTest, TestLazyCores, TestHighlightDedupGrouping,
> TestCollectionAPI, MultiTermTest, SpellCheckCollatorTest, TestUtils,
> BasicDistributedZk2Test, LeaderElectionIntegrationTest, BasicZkTest,
> TestReplicationHandler, LeaderElectionTest, ShardRoutingCustomTest,
> SliceStateUpdateTest, TestDistributedSearch, TestDistributedGrouping,
> TestHashPartitioner, TermVectorComponentDistributedTest, TestJoin,
> TestCoreContainer, TestSolr4Spatial, TestGroupingSearch, TestFunctionQuery,
> DirectUpdateHandlerTest, OverseerCollectionProcessorTest]
>    [junit4] Completed on J1 in 2.78s, 14 tests, 1 error <<< FAILURES!
>
> [...truncated 498 lines...]
> BUILD FAILED
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The
> following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496:
> The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297:
> The following error occurred while executing this line:
> /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921:
> There were test failures: 396 suites, 1640 tests, 1 error, 36 ignored (15
> assumptions)
>
> Total time: 59 minutes 32 seconds
> Build step 'Invoke Ant' marked build as failure
> Description set: Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops
> -XX:+UseParallelGC
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>


-- 
Regards,
Shalin Shekhar Mangar.

[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_20-ea-b11) - Build # 10246 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10246/
Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops -XX:+UseParallelGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerCollectionProcessorTest.testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes

Error Message:
Captured an uncaught exception in thread: Thread[id=5060, name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE, group=TGRP-OverseerCollectionProcessorTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=5060, name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE, group=TGRP-OverseerCollectionProcessorTest]
	at __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729:FD48244C03DACFD]:0)
Caused by: java.util.ConcurrentModificationException
	at __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
	at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
	at java.util.HashMap$KeyIterator.next(HashMap.java:1453)
	at java.util.AbstractCollection.toString(AbstractCollection.java:461)
	at org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
	at org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
	at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11410 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerCollectionProcessorTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerCollectionProcessorTest-39FCE1E3ACCF5729-001/init-core-data-001
   [junit4]   2> 1092908 T5046 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (true)
   [junit4]   2> 1093155 T5046 oas.SolrTestCaseJ4.setUp ###Starting testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
   [junit4]   2> 1093178 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1093179 T5047 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1093197 T5047 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"6",
   [junit4]   2> 	  "maxShardsPerNode":"1"}
   [junit4]   2> 1093198 T5048 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"6",
   [junit4]   2> 	  "maxShardsPerNode":"1"}
   [junit4]   2> 1093198 T5048 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1093199 T5048 oasc.SolrException.log ERROR Collection createcollection of createcollection failed:org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 1, and the number of live nodes is 4. This allows a maximum of 4 to be created. Value of numShards is 6 and value of replicationFactor is 1. This requires 6 shards to be created (higher than the allowed number)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 1093205 T5048 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{Operation createcollection caused exception:=org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 1, and the number of live nodes is 4. This allows a maximum of 4 to be created. Value of numShards is 6 and value of replicationFactor is 1. This requires 6 shards to be created (higher than the allowed number),exception={msg=Cannot create collection mycollection. Value of maxShardsPerNode is 1, and the number of live nodes is 4. This allows a maximum of 4 to be created. Value of numShards is 6 and value of replicationFactor is 1. This requires 6 shards to be created (higher than the allowed number),rspCode=400}}
   [junit4]   2> 1093282 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
   [junit4]   2> 1093285 T5046 oas.SolrTestCaseJ4.setUp ###Starting testNoReplicationEqualNumberOfSlicesPerNode
   [junit4]   2> 1093290 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1093291 T5049 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1093291 T5049 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1093292 T5050 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1093292 T5050 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1093292 T5050 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1093393 T5050 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8] , replicationFactor : 1
   [junit4]   2> 1093393 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1093395 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1093396 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1093396 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1093396 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard5_replica1 as part of slice shard5 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1093397 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard6_replica1 as part of slice shard6 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1093397 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard7_replica1 as part of slice shard7 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1093397 T5050 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard8_replica1 as part of slice shard8 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1093397 T5050 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1093398 T5050 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1093492 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testNoReplicationEqualNumberOfSlicesPerNode
   [junit4]   2> 1093495 T5046 oas.SolrTestCaseJ4.setUp ###Starting testReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
   [junit4]   2> 1093496 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1093496 T5051 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1093497 T5051 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
   [junit4]   2> 1093497 T5052 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
   [junit4]   2> 1093498 T5052 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1093498 T5052 oasc.SolrException.log ERROR Collection createcollection of createcollection failed:org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 2, and the number of live nodes is 3. This allows a maximum of 6 to be created. Value of numShards is 4 and value of replicationFactor is 2. This requires 8 shards to be created (higher than the allowed number)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 1093500 T5052 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{Operation createcollection caused exception:=org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 2, and the number of live nodes is 3. This allows a maximum of 6 to be created. Value of numShards is 4 and value of replicationFactor is 2. This requires 8 shards to be created (higher than the allowed number),exception={msg=Cannot create collection mycollection. Value of maxShardsPerNode is 2, and the number of live nodes is 3. This allows a maximum of 6 to be created. Value of numShards is 4 and value of replicationFactor is 2. This requires 8 shards to be created (higher than the allowed number),rspCode=400}}
   [junit4]   2> 1093597 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
   [junit4]   2> 1093601 T5046 oas.SolrTestCaseJ4.setUp ###Starting testNoReplicationLimitedNodesToCreateOn
   [junit4]   2> 1093603 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1093603 T5053 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1093603 T5053 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"6",
   [junit4]   2> 	  "maxShardsPerNode":"3",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
   [junit4]   2> 1093604 T5054 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"6",
   [junit4]   2> 	  "maxShardsPerNode":"3",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
   [junit4]   2> 1093604 T5054 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1093605 T5054 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1093706 T5054 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5, shard6] , replicationFactor : 1
   [junit4]   2> 1093706 T5054 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1093707 T5054 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1093707 T5054 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1093707 T5054 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1093708 T5054 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard5_replica1 as part of slice shard5 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1093708 T5054 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard6_replica1 as part of slice shard6 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1093708 T5054 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1093709 T5054 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1093804 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testNoReplicationLimitedNodesToCreateOn
   [junit4]   2> 1093807 T5046 oas.SolrTestCaseJ4.setUp ###Starting testReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
   [junit4]   2> 1093808 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1093813 T5055 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1093815 T5055 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
   [junit4]   2> 1093816 T5056 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
   [junit4]   2> 1093816 T5056 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1093817 T5056 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1093917 T5056 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4] , replicationFactor : 2
   [junit4]   2> 1093917 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1093918 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica2 as part of slice shard1 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1093918 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1093918 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica2 as part of slice shard2 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1093918 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1093918 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica2 as part of slice shard3 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1093918 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1093919 T5056 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica2 as part of slice shard4 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1093919 T5056 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1093920 T5056 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1094014 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
   [junit4]   2> 1094018 T5046 oas.SolrTestCaseJ4.setUp ###Starting testNoReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
   [junit4]   2> 1094020 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1094020 T5057 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1094020 T5057 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
   [junit4]   2> 1094021 T5058 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr,localhost:8966_solr"}
   [junit4]   2> 1094021 T5058 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1094021 T5058 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1094122 T5058 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8] , replicationFactor : 1
   [junit4]   2> 1094122 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1094122 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094122 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094123 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1094123 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard5_replica1 as part of slice shard5 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1094123 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard6_replica1 as part of slice shard6 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094123 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard7_replica1 as part of slice shard7 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094124 T5058 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard8_replica1 as part of slice shard8 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1094124 T5058 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1094125 T5058 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1094221 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testNoReplicationEqualNumberOfSlicesPerNodeSendCreateNodesEqualToLiveNodes
   [junit4]   2> 1094226 T5046 oas.SolrTestCaseJ4.setUp ###Starting testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
   [junit4]   2> 1094227 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1094228 T5059 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1094228 T5059 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":null}
   [junit4]   2> 1094228 T5060 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":null}
   [junit4]   2> 1094229 T5060 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1094229 T5060 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1094329 T5060 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4] , replicationFactor : 2
   [junit4]   2> 1094329 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1094330 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica2 as part of slice shard1 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094330 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094330 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica2 as part of slice shard2 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1094330 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1094331 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica2 as part of slice shard3 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094331 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094331 T5060 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica2 as part of slice shard4 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1094331 T5060 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1094333 T5060 oasc.OverseerCollectionProcessor$Runner.resetTaskWithException WARN Resetting task: id, requestid: null, collectionName: mycollection
   [junit4]   2> 1426 T5060 ccr.RandomizedRunner$QueueUncaughtExceptionsHandler.uncaughtException WARNING Uncaught exception in thread: Thread[OverseerThreadFactory-2916-thread-1,5,TGRP-OverseerCollectionProcessorTest] java.util.ConcurrentModificationException
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
   [junit4]   2> 	at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
   [junit4]   2> 	at java.util.HashMap$KeyIterator.next(HashMap.java:1453)
   [junit4]   2> 	at java.util.AbstractCollection.toString(AbstractCollection.java:461)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
   [junit4]   2> 	at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1094428 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=OverseerCollectionProcessorTest -Dtests.method=testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes -Dtests.seed=39FCE1E3ACCF5729 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=sr_CS -Dtests.timezone=Asia/Ust-Nera -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.21s J1 | OverseerCollectionProcessorTest.testReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=5060, name=OverseerThreadFactory-2916-thread-1, state=RUNNABLE, group=TGRP-OverseerCollectionProcessorTest]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729:FD48244C03DACFD]:0)
   [junit4]    > Caused by: java.util.ConcurrentModificationException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([39FCE1E3ACCF5729]:0)
   [junit4]    > 	at java.util.HashMap$HashIterator.nextNode(HashMap.java:1429)
   [junit4]    > 	at java.util.HashMap$KeyIterator.next(HashMap.java:1453)
   [junit4]    > 	at java.util.AbstractCollection.toString(AbstractCollection.java:461)
   [junit4]    > 	at org.apache.solr.cloud.OverseerCollectionProcessor.printTrackingMaps(OverseerCollectionProcessor.java:2714)
   [junit4]    > 	at org.apache.solr.cloud.OverseerCollectionProcessor.access$500(OverseerCollectionProcessor.java:106)
   [junit4]    > 	at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2630)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1094436 T5046 oas.SolrTestCaseJ4.setUp ###Starting testReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
   [junit4]   2> 1094436 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1094437 T5062 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1094438 T5062 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "maxShardsPerNode":"1"}
   [junit4]   2> 1094438 T5063 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "maxShardsPerNode":"1"}
   [junit4]   2> 1094438 T5063 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1094439 T5063 oasc.SolrException.log ERROR Collection createcollection of createcollection failed:org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 1, and the number of live nodes is 4. This allows a maximum of 4 to be created. Value of numShards is 3 and value of replicationFactor is 2. This requires 6 shards to be created (higher than the allowed number)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 1094442 T5063 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{Operation createcollection caused exception:=org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 1, and the number of live nodes is 4. This allows a maximum of 4 to be created. Value of numShards is 3 and value of replicationFactor is 2. This requires 6 shards to be created (higher than the allowed number),exception={msg=Cannot create collection mycollection. Value of maxShardsPerNode is 1, and the number of live nodes is 4. This allows a maximum of 4 to be created. Value of numShards is 3 and value of replicationFactor is 2. This requires 6 shards to be created (higher than the allowed number),rspCode=400}}
   [junit4]   2> 1094537 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testReplicationCollectionNotCreatedDueToMaxShardsPerNodeLimit
   [junit4]   2> 1094541 T5046 oas.SolrTestCaseJ4.setUp ###Starting testNoReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
   [junit4]   2> 1094542 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1094542 T5064 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1094542 T5064 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":null}
   [junit4]   2> 1094543 T5065 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":null}
   [junit4]   2> 1094543 T5065 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1094543 T5065 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1094647 T5065 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5, shard6, shard7, shard8] , replicationFactor : 1
   [junit4]   2> 1094647 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1094648 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1094648 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094648 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094648 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard5_replica1 as part of slice shard5 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1094649 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard6_replica1 as part of slice shard6 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1094649 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard7_replica1 as part of slice shard7 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094649 T5065 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard8_replica1 as part of slice shard8 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094649 T5065 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1094650 T5065 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1094743 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testNoReplicationEqualNumberOfSlicesPerNodeSendNullCreateNodes
   [junit4]   2> 1094747 T5046 oas.SolrTestCaseJ4.setUp ###Starting testReplicationUnequalNumberOfSlicesPerNode
   [junit4]   2> 1094748 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1094749 T5066 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1094749 T5066 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1094749 T5067 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1094750 T5067 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1094750 T5067 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1094850 T5067 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3] , replicationFactor : 2
   [junit4]   2> 1094851 T5067 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094851 T5067 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica2 as part of slice shard1 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094851 T5067 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1094851 T5067 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica2 as part of slice shard2 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1094851 T5067 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1094851 T5067 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica2 as part of slice shard3 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1094852 T5067 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1094853 T5067 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1094950 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testReplicationUnequalNumberOfSlicesPerNode
   [junit4]   2> 1094953 T5046 oas.SolrTestCaseJ4.setUp ###Starting testNoReplicationUnequalNumberOfSlicesPerNode
   [junit4]   2> 1094954 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1094955 T5068 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1094955 T5068 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"6",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1094955 T5069 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"6",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1094956 T5069 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1094956 T5069 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1095056 T5069 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4, shard5, shard6] , replicationFactor : 1
   [junit4]   2> 1095056 T5069 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1095057 T5069 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1095057 T5069 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1095057 T5069 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1095058 T5069 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard5_replica1 as part of slice shard5 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1095058 T5069 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard6_replica1 as part of slice shard6 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1095059 T5069 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1095059 T5069 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1095155 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testNoReplicationUnequalNumberOfSlicesPerNode
   [junit4]   2> 1095159 T5046 oas.SolrTestCaseJ4.setUp ###Starting testReplicationEqualNumberOfSlicesPerNode
   [junit4]   2> 1095160 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1095160 T5070 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1095160 T5070 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1095161 T5071 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"4",
   [junit4]   2> 	  "maxShardsPerNode":"2"}
   [junit4]   2> 1095161 T5071 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1095161 T5071 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1095262 T5071 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3, shard4] , replicationFactor : 2
   [junit4]   2> 1095262 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1095262 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica2 as part of slice shard1 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1095263 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1095263 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica2 as part of slice shard2 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1095263 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8966_solr
   [junit4]   2> 1095263 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica2 as part of slice shard3 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1095264 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica1 as part of slice shard4 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1095264 T5071 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard4_replica2 as part of slice shard4 of collection mycollection on localhost:8965_solr
   [junit4]   2> 1095264 T5071 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1095265 T5071 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1095361 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testReplicationEqualNumberOfSlicesPerNode
   [junit4]   2> 1095364 T5046 oas.SolrTestCaseJ4.setUp ###Starting testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
   [junit4]   2> 1095365 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1095365 T5072 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1095366 T5072 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
   [junit4]   2> 1095366 T5073 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"1",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"8",
   [junit4]   2> 	  "maxShardsPerNode":"2",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr,localhost:8965_solr"}
   [junit4]   2> 1095366 T5073 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1095367 T5073 oasc.SolrException.log ERROR Collection createcollection of createcollection failed:org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 2, and the number of live nodes is 3. This allows a maximum of 6 to be created. Value of numShards is 8 and value of replicationFactor is 1. This requires 8 shards to be created (higher than the allowed number)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.createCollection(OverseerCollectionProcessor.java:2135)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:574)
   [junit4]   2> 		at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2609)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]   2> 		at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 1095368 T5073 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{Operation createcollection caused exception:=org.apache.solr.common.SolrException: Cannot create collection mycollection. Value of maxShardsPerNode is 2, and the number of live nodes is 3. This allows a maximum of 6 to be created. Value of numShards is 8 and value of replicationFactor is 1. This requires 8 shards to be created (higher than the allowed number),exception={msg=Cannot create collection mycollection. Value of maxShardsPerNode is 2, and the number of live nodes is 3. This allows a maximum of 6 to be created. Value of numShards is 8 and value of replicationFactor is 1. This requires 8 shards to be created (higher than the allowed number),rspCode=400}}
   [junit4]   2> 1095466 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testNoReplicationCollectionNotCreatedDueToMaxShardsPerNodeAndNodesToCreateOnLimits
   [junit4]   2> 1095469 T5046 oas.SolrTestCaseJ4.setUp ###Starting testReplicationLimitedNodesToCreateOn
   [junit4]   2> 1095470 T5046 oasc.OverseerCollectionProcessorTest.testTemplate clusterstate 1162041776
   [junit4]   2> 1095470 T5074 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1095471 T5074 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:id message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "maxShardsPerNode":"3",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
   [junit4]   2> 1095471 T5075 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "name":"mycollection",
   [junit4]   2> 	  "collection.configName":"myconfig",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "maxShardsPerNode":"3",
   [junit4]   2> 	  "createNodeSet":"localhost:8963_solr,localhost:8964_solr"}
   [junit4]   2> 1095471 T5075 oasc.OverseerCollectionProcessorTest$1.answer SHARDHANDLER
   [junit4]   2> 1095471 T5075 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/mycollection 
   [junit4]   2> 1095572 T5075 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection mycollection, shardNames [shard1, shard2, shard3] , replicationFactor : 2
   [junit4]   2> 1095572 T5075 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica1 as part of slice shard1 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1095572 T5075 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard1_replica2 as part of slice shard1 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1095573 T5075 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica1 as part of slice shard2 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1095573 T5075 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard2_replica2 as part of slice shard2 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1095573 T5075 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica1 as part of slice shard3 of collection mycollection on localhost:8964_solr
   [junit4]   2> 1095573 T5075 oasc.OverseerCollectionProcessor.createCollection Creating shard mycollection_shard3_replica2 as part of slice shard3 of collection mycollection on localhost:8963_solr
   [junit4]   2> 1095573 T5075 oasc.OverseerCollectionProcessor.createCollection Finished create command on all shards for collection: mycollection
   [junit4]   2> 1095574 T5075 oasc.OverseerCollectionProcessor$Runner.run Overseer Collection Processor: Message id:id complete, response:{success={null=null,null=null,null=null,null=null,null=null,null=null}}
   [junit4]   2> 1095671 T5046 oas.SolrTestCaseJ4.tearDown ###Ending testReplicationLimitedNodesToCreateOn
   [junit4]   2> 1095673 T5046 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: test params are: codec=Lucene40, sim=DefaultSimilarity, locale=sr_CS, timezone=Asia/Ust-Nera
   [junit4]   2> NOTE: Linux 3.8.0-39-generic amd64/Oracle Corporation 1.8.0_20-ea (64-bit)/cpus=8,threads=1,free=155561560,total=386400256
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedExpandComponentTest, ConvertedLegacyTest, TestNumberUtils, TestRecovery, StatsComponentTest, TestRandomFaceting, AlternateDirectoryTest, TestSolrXml, TestClassNameShortening, TestIndexingPerformance, TestSchemaResource, HighlighterConfigTest, SignatureUpdateProcessorFactoryTest, StandardRequestHandlerTest, TestDefaultSimilarityFactory, NumericFieldsTest, OverseerStatusTest, DateFieldTest, TestExpandComponent, TestFastOutputStream, TestIBSimilarityFactory, TestFreeTextSuggestions, OpenCloseCoreStressTest, ResponseHeaderTest, SuggesterWFSTTest, RequestHandlersTest, TestCloudInspectUtil, TestFileDictionaryLookup, HdfsSyncSliceTest, SolrXmlInZkTest, IndexBasedSpellCheckerTest, DeleteReplicaTest, NoCacheHeaderTest, TestCSVResponseWriter, CursorMarkTest, TestFastLRUCache, CurrencyFieldXmlFileTest, TestIndexSearcher, ConnectionManagerTest, ZkSolrClientTest, DisMaxRequestHandlerTest, AtomicUpdatesTest, TestStressReorder, PrimUtilsTest, UniqFieldsUpdateProcessorFactoryTest, TestSolrJ, TestBinaryResponseWriter, DirectSolrSpellCheckerTest, RAMDirectoryFactoryTest, NotRequiredUniqueKeyTest, CoreAdminRequestStatusTest, BadCopyFieldTest, HdfsWriteToMultipleCollectionsTest, TestPartialUpdateDeduplication, TestDocBasedVersionConstraints, CoreContainerCoreInitFailuresTest, OutputWriterTest, JSONWriterTest, TestInfoStreamLogging, SimplePostToolTest, PingRequestHandlerTest, TestHighFrequencyDictionaryFactory, TestAnalyzedSuggestions, TestFieldCollectionResource, RemoteQueryErrorTest, ChaosMonkeySafeLeaderTest, CurrencyFieldOpenExchangeTest, TestArbitraryIndexDir, TestInitQParser, TestSolrQueryParserResource, TestNoOpRegenerator, CollectionsAPIDistributedZkTest, TestCoreDiscovery, TestSolrCoreProperties, HardAutoCommitTest, HdfsCollectionsAPIDistributedZkTest, TestCustomSort, DeleteInactiveReplicaTest, DistributedQueryComponentOptimizationTest, TestSimpleQParserPlugin, TestShardHandlerFactory, SolrIndexConfigTest, TestStressRecovery, OpenExchangeRatesOrgProviderTest, SpellCheckComponentTest, ChaosMonkeyNothingIsSafeTest, TestQueryTypes, DirectSolrConnectionTest, TestLazyCores, TestHighlightDedupGrouping, TestCollectionAPI, MultiTermTest, SpellCheckCollatorTest, TestUtils, BasicDistributedZk2Test, LeaderElectionIntegrationTest, BasicZkTest, TestReplicationHandler, LeaderElectionTest, ShardRoutingCustomTest, SliceStateUpdateTest, TestDistributedSearch, TestDistributedGrouping, TestHashPartitioner, TermVectorComponentDistributedTest, TestJoin, TestCoreContainer, TestSolr4Spatial, TestGroupingSearch, TestFunctionQuery, DirectUpdateHandlerTest, OverseerCollectionProcessorTest]
   [junit4] Completed on J1 in 2.78s, 14 tests, 1 error <<< FAILURES!

[...truncated 498 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921: There were test failures: 396 suites, 1640 tests, 1 error, 36 ignored (15 assumptions)

Total time: 59 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_20-ea-b11 -XX:+UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-4.x-Linux (32bit/jdk1.7.0_55) - Build # 10245 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10245/
Java: 32bit/jdk1.7.0_55 -server -XX:+UseConcMarkSweepGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.OverseerStatusTest.testDistribSearch

Error Message:
createcollection the collection error [Watcher fired on path: null state: SyncConnected type None]

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: createcollection the collection error [Watcher fired on path: null state: SyncConnected type None]
	at __randomizedtesting.SeedInfo.seed([D346A235F13AA4C8:52A02C2D8665C4F4]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.cloud.BasicDistributedZkTest.createCollection(BasicDistributedZkTest.java:624)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1464)
	at org.apache.solr.cloud.OverseerStatusTest.doTest(OverseerStatusTest.java:72)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11768 lines...]
   [junit4] Suite: org.apache.solr.cloud.OverseerStatusTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/init-core-data-001
   [junit4]   2> 1651951 T6197 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /lr/q
   [junit4]   2> 1651955 T6197 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1651955 T6197 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1651956 T6198 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1652056 T6197 oasc.ZkTestServer.run start zk server on port:35099
   [junit4]   2> 1652057 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1673685 T6201 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 21626ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 1673688 T6199 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14628c88de10000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1673689 T6199 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14628c88de10001, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1674906 T6204 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71c53 name:ZooKeeperConnection Watcher:127.0.0.1:35099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1674907 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1674908 T6197 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1674914 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1674915 T6206 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18a8baf name:ZooKeeperConnection Watcher:127.0.0.1:35099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1674916 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1674916 T6197 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1674920 T6197 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1674924 T6197 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1674927 T6197 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1674931 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1674932 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1674937 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1674938 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1674943 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1674943 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1674947 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1674948 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1674953 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1674953 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1674958 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1674959 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1674964 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1674964 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1674969 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1674970 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1674973 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1674974 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1674979 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1674980 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1674984 T6197 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1674984 T6197 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1675234 T6197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1675237 T6197 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:60245
   [junit4]   2> 1675237 T6197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1675238 T6197 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1675239 T6197 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002
   [junit4]   2> 1675240 T6197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/'
   [junit4]   2> 1675266 T6197 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/solr.xml
   [junit4]   2> 1675302 T6197 oasc.CoreContainer.<init> New CoreContainer 31910046
   [junit4]   2> 1675303 T6197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/]
   [junit4]   2> 1675304 T6197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1675304 T6197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1675304 T6197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1675305 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1675305 T6197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1675305 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1675306 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1675306 T6197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1675306 T6197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1675307 T6197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1675307 T6197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1675308 T6197 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1675308 T6197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35099/solr
   [junit4]   2> 1675308 T6197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1675309 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1675310 T6217 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b2146a name:ZooKeeperConnection Watcher:127.0.0.1:35099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1675311 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1675313 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1675313 T6219 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@295f43 name:ZooKeeperConnection Watcher:127.0.0.1:35099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1675314 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1675315 T6197 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1675318 T6197 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1675322 T6197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1675326 T6197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1675331 T6197 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1675337 T6197 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1675339 T6197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60245_lr%2Fq
   [junit4]   2> 1675341 T6197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60245_lr%2Fq
   [junit4]   2> 1675345 T6197 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1675348 T6197 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1675353 T6197 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60245_lr%2Fq
   [junit4]   2> 1675354 T6197 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1675362 T6197 oasc.Overseer.start Overseer (id=91805684246577157-127.0.0.1:60245_lr%2Fq-n_0000000000) starting
   [junit4]   2> 1675367 T6197 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1675383 T6221 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1675384 T6197 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1675387 T6197 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1675392 T6197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1675396 T6220 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1675403 T6222 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1675403 T6222 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1675406 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1675406 T6222 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1675408 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1675409 T6220 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60245/lr/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60245_lr%2Fq",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1675409 T6220 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1675410 T6220 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1675414 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1675414 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1676407 T6222 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1676407 T6222 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1676409 T6222 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1676409 T6222 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1676410 T6222 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1676411 T6222 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/collection1/'
   [junit4]   2> 1676412 T6222 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1676412 T6222 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1676413 T6222 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1676475 T6222 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 1676511 T6222 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1676512 T6222 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1676518 T6222 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1676831 T6222 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1676833 T6222 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1676835 T6222 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1676838 T6222 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1676849 T6222 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1676850 T6222 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1676850 T6222 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-002/collection1/, dataDir=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/control/data/
   [junit4]   2> 1676850 T6222 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5481be
   [junit4]   2> 1676851 T6222 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/control/data
   [junit4]   2> 1676851 T6222 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/control/data/index/
   [junit4]   2> 1676851 T6222 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1676852 T6222 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/control/data/index
   [junit4]   2> 1676852 T6222 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1370338090056513]
   [junit4]   2> 1676852 T6222 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@dc8728 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6363f3),segFN=segments_1,generation=1}
   [junit4]   2> 1676853 T6222 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1676855 T6222 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1676856 T6222 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1676856 T6222 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1676856 T6222 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1676856 T6222 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1676857 T6222 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1676857 T6222 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1676858 T6222 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1676858 T6222 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1676859 T6222 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1676859 T6222 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1676859 T6222 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1676860 T6222 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1676860 T6222 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1676860 T6222 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1676861 T6222 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1676863 T6222 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1676866 T6222 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1676866 T6222 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1676866 T6222 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=57.111328125, floorSegmentMB=0.4677734375, forceMergeDeletesPctAllowed=24.860996340464357, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10214383614950667
   [junit4]   2> 1676867 T6222 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@dc8728 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6363f3),segFN=segments_1,generation=1}
   [junit4]   2> 1676867 T6222 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1676867 T6222 oass.SolrIndexSearcher.<init> Opening Searcher@b12b8f[collection1] main
   [junit4]   2> 1676868 T6222 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1676868 T6222 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1676869 T6222 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1676869 T6222 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1676869 T6222 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1676869 T6222 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1676870 T6222 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1676870 T6222 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1676870 T6222 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1676871 T6222 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1676872 T6223 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@b12b8f[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1676874 T6222 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1676874 T6226 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60245/lr/q collection:control_collection shard:shard1
   [junit4]   2> 1676874 T6197 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1676875 T6197 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1676875 T6226 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1676877 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1676878 T6228 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@d0c4ab name:ZooKeeperConnection Watcher:127.0.0.1:35099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1676878 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1676879 T6197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1676880 T6226 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1676880 T6197 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1676881 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1676882 T6226 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1676882 T6226 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5771 name=collection1 org.apache.solr.core.SolrCore@6b3be0 url=http://127.0.0.1:60245/lr/q/collection1 node=127.0.0.1:60245_lr%2Fq C5771_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:60245/lr/q, core=collection1, node_name=127.0.0.1:60245_lr%2Fq}
   [junit4]   2> 1676882 T6226 C5771 P60245 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:60245/lr/q/collection1/
   [junit4]   2> 1676882 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1676882 T6226 C5771 P60245 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1676883 T6226 C5771 P60245 oasc.SyncStrategy.syncToMe http://127.0.0.1:60245/lr/q/collection1/ has no replicas
   [junit4]   2> 1676883 T6226 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:60245/lr/q/collection1/ shard1
   [junit4]   2> 1676883 T6226 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1676884 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1676887 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1676889 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1676991 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1676991 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1677038 T6226 oasc.ZkController.register We are http://127.0.0.1:60245/lr/q/collection1/ and leader is http://127.0.0.1:60245/lr/q/collection1/
   [junit4]   2> 1677038 T6226 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60245/lr/q
   [junit4]   2> 1677038 T6226 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1677039 T6226 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1677039 T6226 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1677042 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1677042 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1677042 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1677042 T6226 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1677043 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1677044 T6220 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60245/lr/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:60245_lr%2Fq",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1677049 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1677110 T6197 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1677110 T6197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1677112 T6197 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:36518
   [junit4]   2> 1677113 T6197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1677114 T6197 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1677114 T6197 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003
   [junit4]   2> 1677115 T6197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/'
   [junit4]   2> 1677133 T6197 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/solr.xml
   [junit4]   2> 1677150 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1677150 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1677170 T6197 oasc.CoreContainer.<init> New CoreContainer 9459872
   [junit4]   2> 1677171 T6197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/]
   [junit4]   2> 1677172 T6197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1677172 T6197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1677173 T6197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1677173 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1677174 T6197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1677174 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1677174 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1677175 T6197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1677175 T6197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1677176 T6197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1677176 T6197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1677176 T6197 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1677177 T6197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35099/solr
   [junit4]   2> 1677177 T6197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1677178 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1677179 T6239 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@aa9fda name:ZooKeeperConnection Watcher:127.0.0.1:35099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1677180 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1677182 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1677183 T6241 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d1f693 name:ZooKeeperConnection Watcher:127.0.0.1:35099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1677183 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1677190 T6197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1678194 T6197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:36518_lr%2Fq
   [junit4]   2> 1678195 T6197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:36518_lr%2Fq
   [junit4]   2> 1678198 T6228 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1678199 T6219 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1678198 T6241 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1678208 T6242 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1678209 T6242 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1678210 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1678210 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1678211 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1678211 T6242 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1678212 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1678213 T6220 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36518/lr/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36518_lr%2Fq",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1678214 T6220 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 1678214 T6220 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1678222 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1678325 T6241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1678325 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1678325 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1679212 T6242 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1679212 T6242 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1679214 T6242 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1679214 T6242 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1679216 T6242 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1679216 T6242 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/collection1/'
   [junit4]   2> 1679217 T6242 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1679218 T6242 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1679219 T6242 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1679280 T6242 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 1679319 T6242 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1679321 T6242 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1679328 T6242 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1679656 T6242 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1679658 T6242 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1679659 T6242 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1679663 T6242 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1679684 T6242 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1679684 T6242 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1679684 T6242 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-003/collection1/, dataDir=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty1/
   [junit4]   2> 1679685 T6242 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5481be
   [junit4]   2> 1679685 T6242 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty1
   [junit4]   2> 1679686 T6242 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty1/index/
   [junit4]   2> 1679686 T6242 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1679687 T6242 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty1/index
   [junit4]   2> 1679687 T6242 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1370338090056513]
   [junit4]   2> 1679688 T6242 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3bca8c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@babc8),segFN=segments_1,generation=1}
   [junit4]   2> 1679688 T6242 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1679692 T6242 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1679693 T6242 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1679693 T6242 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1679694 T6242 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1679694 T6242 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1679695 T6242 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1679695 T6242 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1679695 T6242 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1679696 T6242 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1679696 T6242 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1679697 T6242 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1679698 T6242 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1679698 T6242 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1679699 T6242 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1679699 T6242 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1679700 T6242 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1679704 T6242 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1679706 T6242 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1679707 T6242 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1679707 T6242 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=57.111328125, floorSegmentMB=0.4677734375, forceMergeDeletesPctAllowed=24.860996340464357, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10214383614950667
   [junit4]   2> 1679708 T6242 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3bca8c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@babc8),segFN=segments_1,generation=1}
   [junit4]   2> 1679708 T6242 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1679708 T6242 oass.SolrIndexSearcher.<init> Opening Searcher@63cf6a[collection1] main
   [junit4]   2> 1679709 T6242 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1679709 T6242 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1679709 T6242 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1679710 T6242 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1679710 T6242 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1679710 T6242 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1679711 T6242 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1679711 T6242 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1679712 T6242 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1679712 T6242 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1679715 T6243 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@63cf6a[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1679717 T6242 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1679718 T6246 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:36518/lr/q collection:collection1 shard:shard2
   [junit4]   2> 1679718 T6197 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1679718 T6197 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1679719 T6246 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 1679723 T6246 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 1679724 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679725 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679725 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679725 T6246 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1679725 T6246 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5772 name=collection1 org.apache.solr.core.SolrCore@18837c0 url=http://127.0.0.1:36518/lr/q/collection1 node=127.0.0.1:36518_lr%2Fq C5772_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:36518/lr/q, core=collection1, node_name=127.0.0.1:36518_lr%2Fq}
   [junit4]   2> 1679725 T6246 C5772 P36518 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:36518/lr/q/collection1/
   [junit4]   2> 1679725 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1679726 T6246 C5772 P36518 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1679726 T6246 C5772 P36518 oasc.SyncStrategy.syncToMe http://127.0.0.1:36518/lr/q/collection1/ has no replicas
   [junit4]   2> 1679726 T6246 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:36518/lr/q/collection1/ shard2
   [junit4]   2> 1679726 T6246 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 1679728 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679731 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679734 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679839 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1679839 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1679839 T6241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1679882 T6246 oasc.ZkController.register We are http://127.0.0.1:36518/lr/q/collection1/ and leader is http://127.0.0.1:36518/lr/q/collection1/
   [junit4]   2> 1679882 T6246 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:36518/lr/q
   [junit4]   2> 1679882 T6246 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1679883 T6246 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1679883 T6246 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1679884 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679885 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679885 T6246 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1679885 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679886 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1679887 T6220 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:36518/lr/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:36518_lr%2Fq",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1679888 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1679947 T6197 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1679948 T6197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1679950 T6197 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:39049
   [junit4]   2> 1679951 T6197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1679951 T6197 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1679952 T6197 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004
   [junit4]   2> 1679952 T6197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/'
   [junit4]   2> 1679965 T6197 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/solr.xml
   [junit4]   2> 1679991 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1679991 T6241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1679991 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1680004 T6197 oasc.CoreContainer.<init> New CoreContainer 26813984
   [junit4]   2> 1680004 T6197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/]
   [junit4]   2> 1680006 T6197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1680006 T6197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1680006 T6197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1680007 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1680007 T6197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1680007 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1680008 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1680008 T6197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1680009 T6197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1680009 T6197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1680009 T6197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1680010 T6197 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1680010 T6197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35099/solr
   [junit4]   2> 1680011 T6197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1680011 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1680013 T6257 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1960e0e name:ZooKeeperConnection Watcher:127.0.0.1:35099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1680013 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1680024 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1680025 T6259 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1d34649 name:ZooKeeperConnection Watcher:127.0.0.1:35099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1680026 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1680033 T6197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1681037 T6197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39049_lr%2Fq
   [junit4]   2> 1681039 T6197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39049_lr%2Fq
   [junit4]   2> 1681043 T6241 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1681043 T6219 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1681044 T6228 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1681045 T6259 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1681057 T6260 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1681058 T6260 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1681065 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1681065 T6260 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1681065 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1681066 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1681067 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1681067 T6220 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39049/lr/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:39049_lr%2Fq",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1681068 T6220 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1681068 T6220 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1681069 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1681171 T6259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1681171 T6241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1681171 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1681171 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682066 T6260 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1682067 T6260 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1682068 T6260 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1682068 T6260 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1682069 T6260 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1682069 T6260 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/collection1/'
   [junit4]   2> 1682070 T6260 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1682071 T6260 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1682071 T6260 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1682136 T6260 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 1682173 T6260 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1682175 T6260 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1682183 T6260 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1682493 T6260 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1682495 T6260 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1682497 T6260 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1682500 T6260 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1682515 T6260 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1682515 T6260 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1682516 T6260 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-004/collection1/, dataDir=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty2/
   [junit4]   2> 1682516 T6260 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5481be
   [junit4]   2> 1682516 T6260 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty2
   [junit4]   2> 1682517 T6260 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty2/index/
   [junit4]   2> 1682517 T6260 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1682517 T6260 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-001/jetty2/index
   [junit4]   2> 1682518 T6260 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1370338090056513]
   [junit4]   2> 1682519 T6260 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f8bd40 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1c68f1b),segFN=segments_1,generation=1}
   [junit4]   2> 1682519 T6260 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1682522 T6260 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1682523 T6260 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1682523 T6260 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1682523 T6260 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1682524 T6260 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1682524 T6260 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1682525 T6260 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1682525 T6260 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1682525 T6260 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1682526 T6260 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1682526 T6260 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1682526 T6260 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1682527 T6260 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1682527 T6260 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1682527 T6260 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1682528 T6260 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1682530 T6260 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1682533 T6260 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1682533 T6260 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1682533 T6260 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=57.111328125, floorSegmentMB=0.4677734375, forceMergeDeletesPctAllowed=24.860996340464357, segmentsPerTier=30.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10214383614950667
   [junit4]   2> 1682534 T6260 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1f8bd40 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1c68f1b),segFN=segments_1,generation=1}
   [junit4]   2> 1682534 T6260 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1682535 T6260 oass.SolrIndexSearcher.<init> Opening Searcher@1931a1b[collection1] main
   [junit4]   2> 1682535 T6260 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1682536 T6260 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1682536 T6260 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1682536 T6260 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1682536 T6260 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1682537 T6260 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1682537 T6260 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1682537 T6260 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1682538 T6260 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1682538 T6260 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1682540 T6261 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1931a1b[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1682542 T6260 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1682543 T6264 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:39049/lr/q collection:collection1 shard:shard1
   [junit4]   2> 1682543 T6197 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1682544 T6264 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1682544 T6197 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1682549 T6264 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1682550 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682550 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682551 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682551 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1682551 T6264 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1682552 T6264 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5773 name=collection1 org.apache.solr.core.SolrCore@a66f98 url=http://127.0.0.1:39049/lr/q/collection1 node=127.0.0.1:39049_lr%2Fq C5773_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:39049/lr/q, core=collection1, node_name=127.0.0.1:39049_lr%2Fq}
   [junit4]   2> 1682552 T6264 C5773 P39049 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:39049/lr/q/collection1/
   [junit4]   2> 1682552 T6264 C5773 P39049 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1682552 T6264 C5773 P39049 oasc.SyncStrategy.syncToMe http://127.0.0.1:39049/lr/q/collection1/ has no replicas
   [junit4]   2> 1682552 T6264 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:39049/lr/q/collection1/ shard1
   [junit4]   2> 1682553 T6264 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1682553 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682555 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682557 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682660 T6241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682660 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682660 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682660 T6259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682706 T6264 oasc.ZkController.register We are http://127.0.0.1:39049/lr/q/collection1/ and leader is http://127.0.0.1:39049/lr/q/collection1/
   [junit4]   2> 1682706 T6264 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:39049/lr/q
   [junit4]   2> 1682706 T6264 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1682707 T6264 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1682707 T6264 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1682708 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682708 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682708 T6264 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1682708 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682709 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1682710 T6220 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39049/lr/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:39049_lr%2Fq",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1682711 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1682794 T6197 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1682795 T6197 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1682796 T6197 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:40634
   [junit4]   2> 1682798 T6197 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1682798 T6197 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1682798 T6197 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-005
   [junit4]   2> 1682799 T6197 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-005/'
   [junit4]   2> 1682811 T6197 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-005/solr.xml
   [junit4]   2> 1682813 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682813 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682814 T6259 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682813 T6241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1682848 T6197 oasc.CoreContainer.<init> New CoreContainer 7418376
   [junit4]   2> 1682848 T6197 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001/tempDir-005/]
   [junit4]   2> 1682850 T6197 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1682850 T6197 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1682851 T6197 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1682851 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1682851 T6197 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1682852 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1682852 T6197 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1682852 T6197 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1682853 T6197 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1682853 T6197 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1682854 T6197 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1682854 T6197 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1682854 T6197 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:35099/solr
   [junit4]   2> 1682855 T6197 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1682855 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1682857 T6275 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18bbde9 name:ZooKeeperConnection Watcher:127.0.0.1:35099 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1682858 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1682862 T6197 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1682863 T6277 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19438cd name:ZooKeeperConnection Watcher:127.0.0.1:35099/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1682863 T6197 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1682875 T6197 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1683879 T6197 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:40634_lr%2Fq
   [junit4]   2> 1683881 T6197 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:40634_lr%2Fq
   [junit4]   2> 1683886 T6219 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1683886 T6228 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1683887 T6277 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1683887 T6241 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1683887 T6259 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 1683895 T6278 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1683896 T6278 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1683898 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1683898 T6278 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1683898 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1683899 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1683901 T6220 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1683902 T6220 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:40634/lr/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:40634_lr%2Fq",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1683902 T6220 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 1683902 T6220 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 1683905 T6219 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1684007 T6219 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1684007 T6241 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1684007 T6228 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1684007 T6277 oas

[...truncated too long message...]

cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 		at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 		at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 	Caused by: org.apache.solr.common.SolrException: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections
   [junit4]   2> 		at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:40)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase.runLeaderProcess(ElectionContext.java:122)
   [junit4]   2> 		... 7 more
   [junit4]   2> 	Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 		at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 		at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1045)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:209)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient$3.execute(SolrZkClient.java:206)
   [junit4]   2> 		at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.exists(SolrZkClient.java:206)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:417)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:386)
   [junit4]   2> 		at org.apache.solr.common.cloud.SolrZkClient.makePath(SolrZkClient.java:373)
   [junit4]   2> 		at org.apache.solr.cloud.ShardLeaderElectionContextBase$1.execute(ElectionContext.java:127)
   [junit4]   2> 		at org.apache.solr.common.util.RetryUtil.retryOnThrowable(RetryUtil.java:34)
   [junit4]   2> 		... 8 more
   [junit4]   2> 	
   [junit4]   2> 1743363 T6297 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1743363 T6297 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1743363 T6297 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1743364 T6297 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.OverseerStatusTest-D346A235F13AA4C8-001
   [junit4]   2> NOTE: test params are: codec=Lucene46: {}, docValues:{}, sim=DefaultSimilarity, locale=es_DO, timezone=Australia/Lord_Howe
   [junit4]   2> NOTE: Linux 3.8.0-39-generic i386/Oracle Corporation 1.7.0_55 (32-bit)/cpus=8,threads=1,free=169653096,total=518979584
   [junit4]   2> NOTE: All tests run in this JVM: [TestComponentsName, DistanceFunctionTest, CursorMarkTest, CachingDirectoryFactoryTest, TestSchemaResource, HdfsCollectionsAPIDistributedZkTest, TermVectorComponentDistributedTest, IndexSchemaRuntimeFieldTest, OpenCloseCoreStressTest, TestCodecSupport, TestSchemaVersionResource, TestAtomicUpdateErrorCases, TestBinaryField, PingRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIBSimilarityFactory, SolrPluginUtilsTest, TestSearcherReuse, AsyncMigrateRouteKeyTest, TestCoreDiscovery, PreAnalyzedUpdateProcessorTest, TestSolrDeletionPolicy2, TestInitQParser, SpellCheckCollatorTest, TestManagedResourceStorage, TestFuzzyAnalyzedSuggestions, TestManagedSchemaFieldResource, TestIndexSearcher, SolrRequestParserTest, PluginInfoTest, HdfsWriteToMultipleCollectionsTest, TestFastWriter, HighlighterTest, TestDistributedSearch, FileUtilsTest, RAMDirectoryFactoryTest, TestFastOutputStream, ConvertedLegacyTest, DocumentAnalysisRequestHandlerTest, CursorPagingTest, TestSolr4Spatial, DebugComponentTest, TestRandomMergePolicy, TestNonDefinedSimilarityFactory, PreAnalyzedFieldTest, BasicDistributedZk2Test, HighlighterConfigTest, TestManagedSynonymFilterFactory, ZkNodePropsTest, DefaultValueUpdateProcessorTest, CollectionsAPIAsyncDistributedZkTest, TestCloudManagedSchemaAddField, TestMultiCoreConfBootstrap, TestFieldTypeCollectionResource, HttpPartitionTest, CoreContainerCoreInitFailuresTest, TestCollationFieldDocValues, HdfsSyncSliceTest, DocValuesMissingTest, TestUpdate, TestStressRecovery, DirectSolrConnectionTest, TestDistributedMissingSort, TestStressLucene, NotRequiredUniqueKeyTest, TestRecoveryHdfs, TestOmitPositions, TestManagedResource, ChaosMonkeyNothingIsSafeTest, TestExceedMaxTermLength, TestStandardQParsers, TestBinaryResponseWriter, TestDistributedGrouping, RequiredFieldsTest, ConnectionManagerTest, QueryResultKeyTest, SpellCheckComponentTest, OpenExchangeRatesOrgProviderTest, TestCollectionAPI, BadIndexSchemaTest, MultiTermTest, SolrCoreCheckLockOnStartupTest, TestMiniSolrCloudCluster, HardAutoCommitTest, TestZkChroot, PrimUtilsTest, SuggesterTSTTest, TestRTGBase, TestIndexingPerformance, TestJmxIntegration, ZkSolrClientTest, DateFieldTest, ChaosMonkeySafeLeaderTest, CollectionsAPIDistributedZkTest, FullSolrCloudDistribCmdsTest, LeaderElectionTest, TestRandomDVFaceting, ZkCLITest, TestFaceting, TestHashPartitioner, TestRealTimeGet, TestReload, SimpleFacetsTest, StatsComponentTest, TestGroupingSearch, TestFunctionQuery, TestLazyCores, DirectUpdateHandlerTest, OverseerCollectionProcessorTest, CurrencyFieldOpenExchangeTest, CurrencyFieldXmlFileTest, SimplePostToolTest, TestExtendedDismaxParser, SuggesterFSTTest, TestFoldingMultitermQuery, DocValuesTest, TestTrie, SuggesterWFSTTest, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, QueryEqualityTest, SortByFunctionTest, DocValuesMultiTest, SolrInfoMBeanTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestQueryUtils, StandardRequestHandlerTest, TestWriterPerf, TestReversedWildcardFilterFactory, PrimitiveFieldTypeTest, XmlUpdateRequestHandlerTest, DocumentBuilderTest, FieldAnalysisRequestHandlerTest, TestArbitraryIndexDir, LoggingHandlerTest, JSONWriterTest, TestCSVResponseWriter, TestAnalyzedSuggestions, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, TestQuerySenderListener, AlternateDirectoryTest, TestQuerySenderNoQuery, BadComponentTest, TestMergePolicyConfig, TestDocSet, NumericFieldsTest, TestElisionMultitermQuery, OutputWriterTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestNumberUtils, TestLMDirichletSimilarityFactory, TestDefaultSimilarityFactory, ScriptEngineTest, DateMathParserTest, TestSystemIdResolver, TestSolrJ, TestUtils, CircularListTest, TestHighlightDedupGrouping, CustomCollectionTest, DeleteShardTest, DistribCursorPagingTest, OverseerRolesTest, OverseerStatusTest]
   [junit4] Completed on J1 in 91.90s, 1 test, 1 error <<< FAILURES!

[...truncated 184 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921: There were test failures: 396 suites, 1640 tests, 1 error, 36 ignored (15 assumptions)

Total time: 70 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 32bit/jdk1.7.0_55 -server -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure