You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Policeman Jenkins Server <je...@thetaphi.de> on 2014/06/12 21:23:09 UTC

[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_05) - Build # 10423 - Failure!

Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10423/
Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseParallelGC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.BasicDistributedZkTest.testDistribSearch

Error Message:
Leader never changed

Stack Trace:
java.lang.AssertionError: Leader never changed
	at __randomizedtesting.SeedInfo.seed([368777E385278B13:B761F9FBF278EB2F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:855)
	at org.apache.solr.cloud.BasicDistributedZkTest.doTest(BasicDistributedZkTest.java:372)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11416 lines...]
   [junit4] Suite: org.apache.solr.cloud.BasicDistributedZkTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/init-core-data-001
   [junit4]   2> 355437 T1993 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 355450 T1993 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 355450 T1993 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 355470 T1994 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 355569 T1993 oasc.ZkTestServer.run start zk server on port:58478
   [junit4]   2> 355581 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 355596 T2000 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2913baeb name:ZooKeeperConnection Watcher:127.0.0.1:58478 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 355596 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 355597 T1993 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 355625 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 355625 T2002 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@150d33ab name:ZooKeeperConnection Watcher:127.0.0.1:58478/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 355626 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 355626 T1993 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 355628 T1993 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 355630 T1993 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 355631 T1993 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 355638 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 355639 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 355642 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 355642 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 355645 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 355645 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 355647 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 355648 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 355650 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 355650 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 355652 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 355653 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 355654 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 355655 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 355658 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 355658 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 355661 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 355661 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 355665 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 355665 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 355667 T1993 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 355668 T1993 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 356075 T1993 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 356077 T1993 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:42083
   [junit4]   2> 356078 T1993 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 356078 T1993 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 356079 T1993 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002
   [junit4]   2> 356079 T1993 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/'
   [junit4]   2> 356096 T1993 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/solr.xml
   [junit4]   2> 356107 T1993 oasc.CoreContainer.<init> New CoreContainer 1402382054
   [junit4]   2> 356107 T1993 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/]
   [junit4]   2> 356108 T1993 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 356108 T1993 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 356109 T1993 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 356109 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 356109 T1993 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 356110 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 356110 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 356110 T1993 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 356111 T1993 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 356111 T1993 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 356112 T1993 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 356112 T1993 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 356113 T1993 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58478/solr
   [junit4]   2> 356113 T1993 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 356138 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 356138 T2013 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6e83e9a8 name:ZooKeeperConnection Watcher:127.0.0.1:58478 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 356139 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 356165 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 356166 T2015 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34b68324 name:ZooKeeperConnection Watcher:127.0.0.1:58478/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 356166 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 356168 T1993 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 356170 T1993 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 356172 T1993 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 356174 T1993 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 356176 T1993 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 356185 T1993 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 356187 T1993 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42083_
   [junit4]   2> 356188 T1993 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42083_
   [junit4]   2> 356191 T1993 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 356194 T1993 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 356200 T1993 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:42083_
   [junit4]   2> 356200 T1993 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 356202 T1993 oasc.Overseer.start Overseer (id=91920762451722243-127.0.0.1:42083_-n_0000000000) starting
   [junit4]   2> 356204 T1993 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 356213 T1993 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 356216 T2017 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 356216 T1993 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 356219 T1993 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 356222 T2016 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 356223 T2018 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 356223 T2018 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 356224 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 356224 T2018 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 356225 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 356225 T2016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42083",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42083_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 356226 T2016 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 356226 T2016 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 356228 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 356228 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 357224 T2018 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 357225 T2018 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 357225 T2018 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 357225 T2018 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 357226 T2018 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 357226 T2018 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/collection1/'
   [junit4]   2> 357227 T2018 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 357227 T2018 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 357227 T2018 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 357249 T2018 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 357258 T2018 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 357259 T2018 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 357265 T2018 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 357438 T2018 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 357439 T2018 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 357439 T2018 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 357445 T2018 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 357446 T2018 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 357459 T2018 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 357462 T2018 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 357465 T2018 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 357466 T2018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 357467 T2018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 357467 T2018 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 357468 T2018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 357468 T2018 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 357468 T2018 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 357468 T2018 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 357469 T2018 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-002/collection1/, dataDir=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/control/data/
   [junit4]   2> 357469 T2018 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@692e353c
   [junit4]   2> 357470 T2018 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/control/data
   [junit4]   2> 357470 T2018 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/control/data/index/
   [junit4]   2> 357470 T2018 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 357470 T2018 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/control/data/index
   [junit4]   2> 357471 T2018 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=36.189453125, floorSegmentMB=1.7841796875, forceMergeDeletesPctAllowed=25.007590612874207, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.12271960390539088
   [junit4]   2> 357471 T2018 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46ef1824 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ffef65a),segFN=segments_1,generation=1}
   [junit4]   2> 357472 T2018 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 357474 T2018 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 357475 T2018 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 357475 T2018 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 357475 T2018 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 357475 T2018 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 357475 T2018 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 357476 T2018 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 357476 T2018 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 357476 T2018 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 357477 T2018 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 357477 T2018 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 357477 T2018 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 357477 T2018 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 357478 T2018 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 357478 T2018 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 357479 T2018 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 357481 T2018 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 357482 T2018 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 357483 T2018 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 357483 T2018 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 357484 T2018 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46ef1824 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ffef65a),segFN=segments_1,generation=1}
   [junit4]   2> 357484 T2018 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 357484 T2018 oass.SolrIndexSearcher.<init> Opening Searcher@196070c2[collection1] main
   [junit4]   2> 357484 T2018 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 357486 T2018 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 357486 T2018 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 357486 T2018 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 357486 T2018 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 357487 T2018 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 357487 T2018 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 357487 T2018 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 357487 T2018 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 357488 T2018 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 357489 T2019 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@196070c2[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 357490 T2018 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 357491 T2022 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42083 collection:control_collection shard:shard1
   [junit4]   2> 357491 T1993 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 357491 T1993 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 357492 T2022 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 357496 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357497 T2024 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10c32034 name:ZooKeeperConnection Watcher:127.0.0.1:58478/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357497 T2022 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 357497 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357499 T1993 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 357500 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357501 T2022 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 357501 T2022 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C233 name=collection1 org.apache.solr.core.SolrCore@2a8f5e52 url=http://127.0.0.1:42083/collection1 node=127.0.0.1:42083_ C233_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:42083, core=collection1, node_name=127.0.0.1:42083_}
   [junit4]   2> 357501 T2022 C233 P42083 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42083/collection1/
   [junit4]   2> 357501 T1993 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 357501 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357501 T2022 C233 P42083 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 357502 T2022 C233 P42083 oasc.SyncStrategy.syncToMe http://127.0.0.1:42083/collection1/ has no replicas
   [junit4]   2> 357502 T2022 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42083/collection1/ shard1
   [junit4]   2> 357502 T2022 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 357512 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357516 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357520 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357626 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 357626 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 357667 T2022 oasc.ZkController.register We are http://127.0.0.1:42083/collection1/ and leader is http://127.0.0.1:42083/collection1/
   [junit4]   2> 357667 T2022 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42083
   [junit4]   2> 357667 T2022 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 357667 T2022 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 357667 T2022 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 357668 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357669 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357669 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357669 T2022 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357670 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 357671 T2016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42083",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42083_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 357673 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 357774 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 357775 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 357804 T1993 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 357805 T1993 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 357807 T1993 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:54178
   [junit4]   2> 357808 T1993 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 357808 T1993 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 357808 T1993 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003
   [junit4]   2> 357808 T1993 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/'
   [junit4]   2> 357821 T1993 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/solr.xml
   [junit4]   2> 357828 T1993 oasc.CoreContainer.<init> New CoreContainer 461914105
   [junit4]   2> 357829 T1993 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/]
   [junit4]   2> 357829 T1993 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 357829 T1993 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 357830 T1993 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 357830 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 357830 T1993 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 357830 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 357831 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 357831 T1993 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 357831 T1993 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 357832 T1993 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 357832 T1993 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 357832 T1993 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 357832 T1993 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58478/solr
   [junit4]   2> 357833 T1993 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 357833 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357835 T2035 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3adb7876 name:ZooKeeperConnection Watcher:127.0.0.1:58478 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357835 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357837 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357840 T2037 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2291e208 name:ZooKeeperConnection Watcher:127.0.0.1:58478/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357840 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357844 T1993 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 358846 T1993 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:54178_
   [junit4]   2> 358847 T1993 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:54178_
   [junit4]   2> 359005 T2015 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 359005 T2037 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 359005 T2024 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 359009 T2038 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 359009 T2038 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 359010 T2038 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 359010 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 359010 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 359011 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 359011 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 359012 T2016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54178",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54178_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 359012 T2016 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 359012 T2016 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 359013 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 359115 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 359115 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 359115 T2037 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 360010 T2038 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 360011 T2038 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 360011 T2038 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 360011 T2038 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 360012 T2038 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 360012 T2038 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/collection1/'
   [junit4]   2> 360012 T2038 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 360013 T2038 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 360013 T2038 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 360028 T2038 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 360036 T2038 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 360037 T2038 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 360041 T2038 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 360126 T2038 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 360127 T2038 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 360127 T2038 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 360130 T2038 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 360131 T2038 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 360138 T2038 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 360140 T2038 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 360141 T2038 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 360142 T2038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 360142 T2038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 360142 T2038 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 360142 T2038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 360143 T2038 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 360143 T2038 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 360143 T2038 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 360143 T2038 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-003/collection1/, dataDir=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty1/
   [junit4]   2> 360143 T2038 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@692e353c
   [junit4]   2> 360144 T2038 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty1
   [junit4]   2> 360144 T2038 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty1/index/
   [junit4]   2> 360144 T2038 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 360144 T2038 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty1/index
   [junit4]   2> 360144 T2038 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=36.189453125, floorSegmentMB=1.7841796875, forceMergeDeletesPctAllowed=25.007590612874207, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.12271960390539088
   [junit4]   2> 360145 T2038 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@651fcb0f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2f662f9a),segFN=segments_1,generation=1}
   [junit4]   2> 360145 T2038 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 360147 T2038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 360147 T2038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 360147 T2038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 360147 T2038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 360147 T2038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 360148 T2038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 360148 T2038 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 360148 T2038 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 360148 T2038 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 360149 T2038 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 360149 T2038 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 360149 T2038 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 360149 T2038 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 360149 T2038 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 360150 T2038 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 360150 T2038 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 360151 T2038 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360152 T2038 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 360152 T2038 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 360153 T2038 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 360153 T2038 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@651fcb0f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2f662f9a),segFN=segments_1,generation=1}
   [junit4]   2> 360153 T2038 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 360153 T2038 oass.SolrIndexSearcher.<init> Opening Searcher@5169049d[collection1] main
   [junit4]   2> 360154 T2038 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 360154 T2038 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 360154 T2038 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 360155 T2038 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 360155 T2038 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 360155 T2038 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 360155 T2038 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 360155 T2038 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 360156 T2038 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 360156 T2038 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 360158 T2039 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5169049d[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 360159 T2038 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 360159 T2042 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:54178 collection:collection1 shard:shard2
   [junit4]   2> 360160 T1993 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 360160 T1993 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 360160 T2042 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 360163 T2042 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 360164 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360164 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360165 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360164 T2042 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 360165 T2042 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C234 name=collection1 org.apache.solr.core.SolrCore@222f535e url=http://127.0.0.1:54178/collection1 node=127.0.0.1:54178_ C234_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:54178, core=collection1, node_name=127.0.0.1:54178_}
   [junit4]   2> 360165 T2042 C234 P54178 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:54178/collection1/
   [junit4]   2> 360165 T2042 C234 P54178 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 360165 T2042 C234 P54178 oasc.SyncStrategy.syncToMe http://127.0.0.1:54178/collection1/ has no replicas
   [junit4]   2> 360165 T2042 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:54178/collection1/ shard2
   [junit4]   2> 360165 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 360166 T2042 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 360167 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360169 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360182 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360283 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 360283 T2037 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 360283 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 360320 T2042 oasc.ZkController.register We are http://127.0.0.1:54178/collection1/ and leader is http://127.0.0.1:54178/collection1/
   [junit4]   2> 360320 T2042 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:54178
   [junit4]   2> 360320 T2042 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 360320 T2042 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 360320 T2042 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 360321 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360321 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360322 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360322 T2042 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 360322 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 360323 T2016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:54178",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:54178_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 360324 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 360390 T1993 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 360391 T1993 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 360393 T1993 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:49001
   [junit4]   2> 360393 T1993 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 360394 T1993 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 360394 T1993 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004
   [junit4]   2> 360394 T1993 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/'
   [junit4]   2> 360405 T1993 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/solr.xml
   [junit4]   2> 360412 T1993 oasc.CoreContainer.<init> New CoreContainer 1179899115
   [junit4]   2> 360413 T1993 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/]
   [junit4]   2> 360413 T1993 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 360413 T1993 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 360413 T1993 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 360414 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 360414 T1993 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 360414 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 360414 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 360414 T1993 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 360415 T1993 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 360415 T1993 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 360416 T1993 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 360422 T1993 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 360423 T1993 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58478/solr
   [junit4]   2> 360423 T1993 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 360423 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 360425 T2053 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e27c71c name:ZooKeeperConnection Watcher:127.0.0.1:58478 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 360425 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 360425 T2037 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 360425 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 360425 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 360427 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 360428 T2055 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@15d9408f name:ZooKeeperConnection Watcher:127.0.0.1:58478/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 360428 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 360432 T1993 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 361434 T1993 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49001_
   [junit4]   2> 361435 T1993 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49001_
   [junit4]   2> 361448 T2015 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 361448 T2024 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 361448 T2037 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 361448 T2055 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 361454 T2056 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 361454 T2056 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 361455 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 361455 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 361455 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 361455 T2056 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 361456 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 361457 T2016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49001",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49001_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 361458 T2016 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 361458 T2016 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 361460 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 361562 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 361562 T2037 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 361562 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 361562 T2055 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362456 T2056 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 362456 T2056 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 362457 T2056 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 362457 T2056 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 362457 T2056 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 362458 T2056 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/collection1/'
   [junit4]   2> 362458 T2056 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 362458 T2056 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 362458 T2056 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 362474 T2056 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 362482 T2056 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 362483 T2056 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 362487 T2056 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 362574 T2056 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 362574 T2056 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 362574 T2056 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 362577 T2056 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 362578 T2056 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 362585 T2056 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 362587 T2056 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 362589 T2056 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 362589 T2056 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 362589 T2056 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 362590 T2056 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 362590 T2056 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 362590 T2056 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 362590 T2056 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 362591 T2056 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 362591 T2056 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-004/collection1/, dataDir=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty2/
   [junit4]   2> 362591 T2056 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@692e353c
   [junit4]   2> 362592 T2056 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty2
   [junit4]   2> 362592 T2056 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty2/index/
   [junit4]   2> 362592 T2056 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 362593 T2056 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-001/jetty2/index
   [junit4]   2> 362593 T2056 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=36, maxMergedSegmentMB=36.189453125, floorSegmentMB=1.7841796875, forceMergeDeletesPctAllowed=25.007590612874207, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.12271960390539088
   [junit4]   2> 362593 T2056 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7b7537eb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38090ea),segFN=segments_1,generation=1}
   [junit4]   2> 362594 T2056 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 362595 T2056 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 362595 T2056 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 362596 T2056 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 362596 T2056 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 362596 T2056 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 362596 T2056 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 362596 T2056 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 362597 T2056 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 362597 T2056 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 362597 T2056 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 362597 T2056 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 362597 T2056 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 362597 T2056 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 362598 T2056 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 362598 T2056 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 362598 T2056 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 362599 T2056 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 362600 T2056 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 362601 T2056 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 362601 T2056 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 362601 T2056 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7b7537eb lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38090ea),segFN=segments_1,generation=1}
   [junit4]   2> 362601 T2056 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 362602 T2056 oass.SolrIndexSearcher.<init> Opening Searcher@6d74be02[collection1] main
   [junit4]   2> 362602 T2056 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 362602 T2056 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 362603 T2056 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 362603 T2056 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 362603 T2056 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 362603 T2056 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 362604 T2056 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 362604 T2056 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 362604 T2056 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 362604 T2056 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 362606 T2057 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d74be02[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 362607 T2056 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 362607 T2060 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49001 collection:collection1 shard:shard1
   [junit4]   2> 362607 T1993 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0
   [junit4]   2> 362608 T1993 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 362608 T2060 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 362611 T2060 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 362612 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362612 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362612 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362612 T2060 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 362612 T2060 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 362613 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C235 name=collection1 org.apache.solr.core.SolrCore@69e9bd0c url=http://127.0.0.1:49001/collection1 node=127.0.0.1:49001_ C235_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:49001, core=collection1, node_name=127.0.0.1:49001_}
   [junit4]   2> 362613 T2060 C235 P49001 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49001/collection1/
   [junit4]   2> 362613 T2060 C235 P49001 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 362613 T2060 C235 P49001 oasc.SyncStrategy.syncToMe http://127.0.0.1:49001/collection1/ has no replicas
   [junit4]   2> 362613 T2060 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49001/collection1/ shard1
   [junit4]   2> 362614 T2060 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 362614 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362616 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362620 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362722 T2055 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362722 T2037 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362722 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362722 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362767 T2060 oasc.ZkController.register We are http://127.0.0.1:49001/collection1/ and leader is http://127.0.0.1:49001/collection1/
   [junit4]   2> 362767 T2060 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49001
   [junit4]   2> 362767 T2060 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 362767 T2060 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 362767 T2060 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 362769 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362769 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362769 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362769 T2060 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 362769 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 362770 T2016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49001",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49001_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 362771 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 362834 T1993 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 362835 T1993 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 362836 T1993 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:60734
   [junit4]   2> 362837 T1993 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 362837 T1993 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 362838 T1993 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-005
   [junit4]   2> 362838 T1993 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-005/'
   [junit4]   2> 362851 T1993 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-005/solr.xml
   [junit4]   2> 362856 T1993 oasc.CoreContainer.<init> New CoreContainer 1065808751
   [junit4]   2> 362857 T1993 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/tempDir-005/]
   [junit4]   2> 362857 T1993 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 362857 T1993 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 362858 T1993 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 362858 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 362858 T1993 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 362858 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 362858 T1993 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 362859 T1993 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 362859 T1993 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 362859 T1993 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 362860 T1993 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 362860 T1993 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 362860 T1993 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:58478/solr
   [junit4]   2> 362860 T1993 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 362861 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 362862 T2071 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1befcd86 name:ZooKeeperConnection Watcher:127.0.0.1:58478 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 362863 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 362864 T1993 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 362865 T2073 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2cdffb8f name:ZooKeeperConnection Watcher:127.0.0.1:58478/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 362865 T1993 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 362871 T1993 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 362884 T2055 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362884 T2024 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362884 T2073 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362884 T2037 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 362884 T2015 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 363885 T1993 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60734_
   [junit4]   2> 363886 T1993 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60734_
   [junit4]   2> 363887 T2015 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 363887 T2055 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 363887 T2037 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 363887 T2073 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 363887 T2024 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 363890 T2074 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 363890 T2074 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 363891 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 363891 T2074 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 363891 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 363891 T2015 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 363892 T2016 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 363893 T2016 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60734",


[...truncated too long message...]

 IndexWriter with IndexWriterCloser
   [junit4]   2> 412234 T2093 oasc.SolrCore.closeSearcher [collection2] Closing main searcher on request.
   [junit4]   2> 412234 T2093 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 412234 T2093 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/collection2-004/index [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/collection2-004/index;done=false>>]
   [junit4]   2> 412234 T2093 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/collection2-004/index
   [junit4]   2> 412234 T2093 oasc.CachingDirectoryFactory.closeCacheValue looking to close /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/collection2-004 [CachedDir<<refCount=0;path=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/collection2-004;done=false>>]
   [junit4]   2> 412235 T2093 oasc.CachingDirectoryFactory.close Closing directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001/collection2-004
   [junit4]   2> 412235 T2093 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection3/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 412235 T2093 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 412235 T2093 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 412236 T2093 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.BasicDistributedZkTest-368777E385278B13-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene49), sim=DefaultSimilarity, locale=el, timezone=Europe/Prague
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_05 (64-bit)/cpus=8,threads=1,free=155763568,total=238551040
   [junit4]   2> NOTE: All tests run in this JVM: [UpdateRequestProcessorFactoryTest, OverseerTest, TestUniqueKeyFieldResource, QueryEqualityTest, TestDistribDocBasedVersion, TestLFUCache, TestNoOpRegenerator, SoftAutoCommitTest, TermVectorComponentTest, CircularListTest, WordBreakSolrSpellCheckerTest, TestStressUserVersions, RecoveryZkTest, UpdateParamsTest, CoreMergeIndexesAdminHandlerTest, HdfsBasicDistributedZk2Test, FastVectorHighlighterTest, SearchHandlerTest, TestSuggestSpellingConverter, HdfsUnloadDistributedZkTest, TestConfigSets, MergeStrategyTest, TestWriterPerf, UUIDFieldTest, TestBadConfig, MoreLikeThisHandlerTest, TestSchemaNameResource, SpatialFilterTest, TestFileDictionaryLookup, SliceStateTest, TestManagedResourceStorage, TestSolrXml, TestHighFrequencyDictionaryFactory, HdfsBasicDistributedZkTest, TestCollapseQParserPlugin, TestPseudoReturnFields, TestSchemaSimilarityResource, BadComponentTest, SystemInfoHandlerTest, TestGroupingSearch, TestLMJelinekMercerSimilarityFactory, DeleteShardTest, TestLRUCache, TestCharFilters, QueryParsingTest, TestDocBasedVersionConstraints, SolrTestCaseJ4Test, BlockDirectoryTest, TestSystemIdResolver, SimpleFacetsTest, PrimitiveFieldTypeTest, AliasIntegrationTest, AnalysisAfterCoreReloadTest, ResourceLoaderTest, DocValuesMissingTest, TestSearchPerf, SliceStateUpdateTest, SOLR749Test, CursorMarkTest, TestCursorMarkWithoutUniqueKey, BinaryUpdateRequestHandlerTest, ResponseHeaderTest, SuggesterTest, DOMUtilTest, URLClassifyProcessorTest, TestRecoveryHdfs, TestSolrQueryParser, TestSolrXmlPersistor, DistributedTermsComponentTest, TestReRankQParserPlugin, TestStressVersions, JsonLoaderTest, LukeRequestHandlerTest, TestPerFieldSimilarity, PolyFieldTest, PeerSyncTest, TestFieldSortValues, RegexBoostProcessorTest, TestValueSourceCache, ClusterStateTest, TestReversedWildcardFilterFactory, TestSurroundQueryParser, DocumentBuilderTest, TestBM25SimilarityFactory, TestDocSet, SynonymTokenizerTest, TestFreeTextSuggestions, TestPHPSerializedResponseWriter, TestFunctionQuery, AutoCommitTest, CacheHeaderTest, CopyFieldTest, TestSolrXMLSerializer, TestCloudManagedSchema, ConnectionManagerTest, EnumFieldTest, BasicDistributedZkTest]
   [junit4] Completed on J0 in 57.56s, 1 test, 1 failure <<< FAILURES!

[...truncated 786 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921: There were test failures: 404 suites, 1648 tests, 1 failure, 37 ignored (16 assumptions)

Total time: 65 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseParallelGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_05) - Build # 10426 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10426/
Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseG1GC

3 tests failed.
REGRESSION:  org.apache.solr.cloud.HttpPartitionTest.testDistribSearch

Error Message:
createcollection the collection error [Watcher fired on path: null state: SyncConnected type None]

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: createcollection the collection error [Watcher fired on path: null state: SyncConnected type None]
	at __randomizedtesting.SeedInfo.seed([28A2A804290B2957:A944261C5E54496B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1508)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1468)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1527)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1464)
	at org.apache.solr.cloud.HttpPartitionTest.testRf3(HttpPartitionTest.java:223)
	at org.apache.solr.cloud.HttpPartitionTest.doTest(HttpPartitionTest.java:145)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.ShardSplitTest.testDistribSearch

Error Message:
shard1_0 is not active expected:<[active]> but was:<[recovery]>

Stack Trace:
org.junit.ComparisonFailure: shard1_0 is not active expected:<[active]> but was:<[recovery]>
	at __randomizedtesting.SeedInfo.seed([28A2A804290B2957:A944261C5E54496B]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.apache.solr.cloud.ShardSplitTest.checkDocCountsAndShardStates(ShardSplitTest.java:433)
	at org.apache.solr.cloud.ShardSplitTest.splitByUniqueKeyTest(ShardSplitTest.java:240)
	at org.apache.solr.cloud.ShardSplitTest.doTest(ShardSplitTest.java:104)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([28A2A804290B2957:A944261C5E54496B]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:659)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1149)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11097 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/init-core-data-001
   [junit4]   2> 535170 T1643 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 535172 T1643 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 535173 T1643 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 535173 T1644 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 535273 T1643 oasc.ZkTestServer.run start zk server on port:51084
   [junit4]   2> 535274 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535431 T1650 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7229c18e name:ZooKeeperConnection Watcher:127.0.0.1:51084 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535432 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535432 T1643 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 535438 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535440 T1652 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fc3e674 name:ZooKeeperConnection Watcher:127.0.0.1:51084/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535440 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535441 T1643 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 535444 T1643 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 535448 T1643 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 535450 T1643 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 535452 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 535453 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 535457 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 535458 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 535461 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 535462 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 535465 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 535465 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 535467 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 535468 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 535472 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 535472 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 535475 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 535476 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 535479 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 535479 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 535481 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 535482 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 535485 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 535486 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 535488 T1643 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 535489 T1643 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 535720 T1643 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 535739 T1643 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:37779
   [junit4]   2> 535740 T1643 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 535740 T1643 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 535741 T1643 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002
   [junit4]   2> 535742 T1643 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/'
   [junit4]   2> 535767 T1643 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/solr.xml
   [junit4]   2> 535774 T1643 oasc.CoreContainer.<init> New CoreContainer 710906667
   [junit4]   2> 535774 T1643 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/]
   [junit4]   2> 535774 T1643 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 535775 T1643 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 535775 T1643 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 535775 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 535775 T1643 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 535776 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 535776 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 535776 T1643 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 535776 T1643 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 535777 T1643 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 535777 T1643 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 535777 T1643 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 535778 T1643 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51084/solr
   [junit4]   2> 535778 T1643 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 535779 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535781 T1663 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@101660e7 name:ZooKeeperConnection Watcher:127.0.0.1:51084 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535782 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535785 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 535787 T1665 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f98bbb name:ZooKeeperConnection Watcher:127.0.0.1:51084/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 535787 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 535789 T1643 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 535793 T1643 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 535797 T1643 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 535802 T1643 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 535807 T1643 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 535816 T1643 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 535819 T1643 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37875_
   [junit4]   2> 535821 T1643 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37875_
   [junit4]   2> 535825 T1643 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 535828 T1643 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 535834 T1643 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37875_
   [junit4]   2> 535835 T1643 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 535838 T1643 oasc.Overseer.start Overseer (id=91922496264798211-127.0.0.1:37875_-n_0000000000) starting
   [junit4]   2> 535843 T1643 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 535857 T1667 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 535858 T1643 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 535861 T1643 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 535864 T1643 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 535865 T1666 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 535867 T1668 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 535867 T1668 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 535867 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 535867 T1668 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 535868 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 535869 T1666 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37875",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37875_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 535869 T1666 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 535869 T1666 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 535870 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 535871 T1665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 536868 T1668 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 536868 T1668 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 536869 T1668 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 536869 T1668 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 536870 T1668 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 536870 T1668 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/collection1/'
   [junit4]   2> 536870 T1668 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 536870 T1668 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 536871 T1668 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 536888 T1668 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 536896 T1668 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 536897 T1668 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 536901 T1668 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 536985 T1668 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 536986 T1668 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 536986 T1668 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 536990 T1668 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 536991 T1668 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 536999 T1668 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 537001 T1668 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 537002 T1668 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 537003 T1668 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 537003 T1668 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 537003 T1668 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 537004 T1668 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 537004 T1668 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 537004 T1668 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 537005 T1668 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 537005 T1668 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-002/collection1/, dataDir=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/control/data/
   [junit4]   2> 537005 T1668 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4fc763fa
   [junit4]   2> 537005 T1668 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/control/data
   [junit4]   2> 537006 T1668 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/control/data/index/
   [junit4]   2> 537006 T1668 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 537006 T1668 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/control/data/index
   [junit4]   2> 537006 T1668 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=71.505859375, floorSegmentMB=1.7490234375, forceMergeDeletesPctAllowed=5.424670760772601, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 537007 T1668 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2638c5a6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36cf5051),segFN=segments_1,generation=1}
   [junit4]   2> 537007 T1668 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 537009 T1668 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 537009 T1668 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 537009 T1668 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 537009 T1668 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 537009 T1668 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 537010 T1668 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 537010 T1668 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 537010 T1668 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 537010 T1668 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 537010 T1668 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 537011 T1668 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 537011 T1668 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 537011 T1668 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 537011 T1668 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 537012 T1668 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 537012 T1668 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 537013 T1668 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 537014 T1668 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 537015 T1668 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 537015 T1668 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=3.4560546875, floorSegmentMB=2.0302734375, forceMergeDeletesPctAllowed=20.00552080589751, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4540328940412207
   [junit4]   2> 537016 T1668 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2638c5a6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36cf5051),segFN=segments_1,generation=1}
   [junit4]   2> 537016 T1668 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 537016 T1668 oass.SolrIndexSearcher.<init> Opening Searcher@61e08c26[collection1] main
   [junit4]   2> 537016 T1668 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 537017 T1668 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 537017 T1668 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 537018 T1668 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 537018 T1668 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 537018 T1668 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 537018 T1668 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 537018 T1668 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 537019 T1668 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 537019 T1668 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 537020 T1669 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@61e08c26[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 537021 T1668 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 537021 T1672 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37875 collection:control_collection shard:shard1
   [junit4]   2> 537021 T1643 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 537022 T1672 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 537022 T1643 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 537025 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 537025 T1672 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 537030 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537030 T1672 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 537030 T1672 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C538 name=collection1 org.apache.solr.core.SolrCore@1d24d0c7 url=http://127.0.0.1:37875/collection1 node=127.0.0.1:37875_ C538_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:37875, core=collection1, node_name=127.0.0.1:37875_}
   [junit4]   2> 537030 T1672 C538 P37875 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37875/collection1/
   [junit4]   2> 537031 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 537031 T1672 C538 P37875 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 537031 T1672 C538 P37875 oasc.SyncStrategy.syncToMe http://127.0.0.1:37875/collection1/ has no replicas
   [junit4]   2> 537031 T1672 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37875/collection1/ shard1
   [junit4]   2> 537031 T1675 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14ae9554 name:ZooKeeperConnection Watcher:127.0.0.1:51084/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 537031 T1672 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 537032 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 537036 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537036 T1643 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 537038 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537038 T1643 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 537041 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537143 T1675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 537143 T1665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 537189 T1672 oasc.ZkController.register We are http://127.0.0.1:37875/collection1/ and leader is http://127.0.0.1:37875/collection1/
   [junit4]   2> 537189 T1672 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37875
   [junit4]   2> 537189 T1672 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 537189 T1672 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 537189 T1672 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 537191 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537191 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537191 T1672 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 537191 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537192 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 537192 T1666 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37875",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37875_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 537194 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 537277 T1643 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 537278 T1643 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 537280 T1643 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:59602
   [junit4]   2> 537281 T1643 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 537281 T1643 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 537281 T1643 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003
   [junit4]   2> 537282 T1643 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/'
   [junit4]   2> 537294 T1643 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/solr.xml
   [junit4]   2> 537295 T1675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 537295 T1665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 537318 T1643 oasc.CoreContainer.<init> New CoreContainer 34999867
   [junit4]   2> 537319 T1643 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/]
   [junit4]   2> 537319 T1643 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 537319 T1643 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 537320 T1643 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 537320 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 537320 T1643 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 537320 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 537321 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 537321 T1643 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 537321 T1643 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 537322 T1643 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 537322 T1643 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 537322 T1643 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 537322 T1643 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51084/solr
   [junit4]   2> 537323 T1643 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 537323 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 537325 T1686 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@675ace59 name:ZooKeeperConnection Watcher:127.0.0.1:51084 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 537326 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 537328 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 537330 T1688 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@345d871b name:ZooKeeperConnection Watcher:127.0.0.1:51084/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 537330 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 537334 T1643 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 538336 T1643 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42332_
   [junit4]   2> 538337 T1643 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42332_
   [junit4]   2> 538343 T1675 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 538343 T1665 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 538344 T1688 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 538347 T1689 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 538348 T1689 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 538348 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 538348 T1689 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 538348 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 538349 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 538349 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 538350 T1666 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42332",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42332_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 538350 T1666 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1, shard2]
   [junit4]   2> 538350 T1666 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 538352 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 538454 T1675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 538454 T1665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 538454 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 539349 T1689 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 539349 T1689 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 539350 T1689 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 539350 T1689 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 539351 T1689 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 539351 T1689 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/collection1/'
   [junit4]   2> 539351 T1689 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 539352 T1689 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 539352 T1689 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 539373 T1689 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 539381 T1689 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 539382 T1689 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 539387 T1689 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 539471 T1689 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 539472 T1689 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 539472 T1689 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 539476 T1689 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 539477 T1689 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 539484 T1689 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 539486 T1689 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 539487 T1689 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 539488 T1689 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 539488 T1689 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 539489 T1689 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 539489 T1689 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 539489 T1689 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 539489 T1689 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 539490 T1689 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 539490 T1689 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-003/collection1/, dataDir=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty1/
   [junit4]   2> 539490 T1689 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4fc763fa
   [junit4]   2> 539491 T1689 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty1
   [junit4]   2> 539491 T1689 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty1/index/
   [junit4]   2> 539491 T1689 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 539491 T1689 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty1/index
   [junit4]   2> 539491 T1689 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=71.505859375, floorSegmentMB=1.7490234375, forceMergeDeletesPctAllowed=5.424670760772601, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 539492 T1689 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f3f67a9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@46de0947),segFN=segments_1,generation=1}
   [junit4]   2> 539492 T1689 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 539494 T1689 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 539494 T1689 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 539494 T1689 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 539495 T1689 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 539495 T1689 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 539495 T1689 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 539495 T1689 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 539495 T1689 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 539495 T1689 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 539496 T1689 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 539496 T1689 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 539496 T1689 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 539496 T1689 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 539497 T1689 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 539497 T1689 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 539497 T1689 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 539499 T1689 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 539500 T1689 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 539500 T1689 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 539500 T1689 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=3.4560546875, floorSegmentMB=2.0302734375, forceMergeDeletesPctAllowed=20.00552080589751, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4540328940412207
   [junit4]   2> 539501 T1689 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5f3f67a9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@46de0947),segFN=segments_1,generation=1}
   [junit4]   2> 539501 T1689 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 539501 T1689 oass.SolrIndexSearcher.<init> Opening Searcher@9e3f209[collection1] main
   [junit4]   2> 539501 T1689 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 539502 T1689 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 539502 T1689 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 539502 T1689 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 539503 T1689 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 539503 T1689 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 539503 T1689 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 539503 T1689 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 539503 T1689 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 539504 T1689 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 539505 T1690 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9e3f209[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 539507 T1689 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 539507 T1693 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42332 collection:collection1 shard:shard2
   [junit4]   2> 539507 T1643 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 539508 T1693 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 539508 T1643 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 539511 T1693 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 539512 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539512 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539512 T1693 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 539512 T1693 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 539512 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> ASYNC  NEW_CORE C539 name=collection1 org.apache.solr.core.SolrCore@79de98e8 url=http://127.0.0.1:42332/collection1 node=127.0.0.1:42332_ C539_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:42332, core=collection1, node_name=127.0.0.1:42332_}
   [junit4]   2> 539512 T1693 C539 P42332 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42332/collection1/
   [junit4]   2> 539513 T1693 C539 P42332 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 539513 T1693 C539 P42332 oasc.SyncStrategy.syncToMe http://127.0.0.1:42332/collection1/ has no replicas
   [junit4]   2> 539513 T1693 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42332/collection1/ shard2
   [junit4]   2> 539513 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 539513 T1693 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 539515 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539516 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539519 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539621 T1675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 539621 T1665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 539621 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 539667 T1693 oasc.ZkController.register We are http://127.0.0.1:42332/collection1/ and leader is http://127.0.0.1:42332/collection1/
   [junit4]   2> 539667 T1693 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42332
   [junit4]   2> 539668 T1693 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 539668 T1693 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 539668 T1693 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 539669 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539669 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539669 T1693 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 539669 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539670 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 539671 T1666 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42332",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:42332_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 539673 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 539762 T1643 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 539763 T1643 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 539765 T1643 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:59179
   [junit4]   2> 539766 T1643 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 539766 T1643 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 539766 T1643 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004
   [junit4]   2> 539767 T1643 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/'
   [junit4]   2> 539774 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 539774 T1665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 539774 T1675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 539781 T1643 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/solr.xml
   [junit4]   2> 539788 T1643 oasc.CoreContainer.<init> New CoreContainer 1095116504
   [junit4]   2> 539789 T1643 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/]
   [junit4]   2> 539789 T1643 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 539790 T1643 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 539790 T1643 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 539790 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 539790 T1643 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 539790 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 539791 T1643 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 539791 T1643 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 539791 T1643 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 539792 T1643 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 539792 T1643 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 539793 T1643 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 539793 T1643 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:51084/solr
   [junit4]   2> 539793 T1643 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 539794 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 539796 T1705 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a4b11d4 name:ZooKeeperConnection Watcher:127.0.0.1:51084 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 539797 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 539799 T1643 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 539800 T1707 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@311540e3 name:ZooKeeperConnection Watcher:127.0.0.1:51084/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 539800 T1643 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 539804 T1643 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 540806 T1643 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37136_
   [junit4]   2> 540807 T1643 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37136_
   [junit4]   2> 540813 T1675 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 540813 T1665 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 540813 T1707 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 540813 T1688 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 540817 T1708 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 540817 T1708 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 540818 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 540818 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 540818 T1708 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 540818 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 540819 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 540819 T1666 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37136",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:37136_",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 540819 T1666 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=2
   [junit4]   2> 540819 T1666 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 540823 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 540924 T1675 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 540924 T1707 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 540924 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 540924 T1665 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 541818 T1708 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 541819 T1708 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 541820 T1708 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 541820 T1708 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 541822 T1708 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 541822 T1708 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/collection1/'
   [junit4]   2> 541823 T1708 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 541823 T1708 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 541824 T1708 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 541851 T1708 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 541861 T1708 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 541863 T1708 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 541867 T1708 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 541967 T1708 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 541968 T1708 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 541968 T1708 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 541971 T1708 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 541972 T1708 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 541986 T1708 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 541988 T1708 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 541990 T1708 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 541990 T1708 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 541990 T1708 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 541991 T1708 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 541992 T1708 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 541992 T1708 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 541992 T1708 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 541993 T1708 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 541993 T1708 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-004/collection1/, dataDir=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty2/
   [junit4]   2> 541993 T1708 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4fc763fa
   [junit4]   2> 541994 T1708 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty2
   [junit4]   2> 541995 T1708 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty2/index/
   [junit4]   2> 541995 T1708 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 541995 T1708 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.HttpPartitionTest-28A2A804290B2957-001/tempDir-001/jetty2/index
   [junit4]   2> 541996 T1708 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=29, maxMergeAtOnceExplicit=33, maxMergedSegmentMB=71.505859375, floorSegmentMB=1.7490234375, forceMergeDeletesPctAllowed=5.424670760772601, segmentsPerTier=48.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 541997 T1708 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbb2b51 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@436b308c),segFN=segments_1,generation=1}
   [junit4]   2> 541997 T1708 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 542002 T1708 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 542003 T1708 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 542003 T1708 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 542003 T1708 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 542004 T1708 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 542004 T1708 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 542004 T1708 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 542004 T1708 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 542005 T1708 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 542005 T1708 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 542005 T1708 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 542005 T1708 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 542006 T1708 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 542006 T1708 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 542006 T1708 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 542007 T1708 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 542008 T1708 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 542009 T1708 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 542009 T1708 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 542010 T1708 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=3.4560546875, floorSegmentMB=2.0302734375, forceMergeDeletesPctAllowed=20.00552080589751, segmentsPerTier=20.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4540328940412207
   [junit4]   2> 542010 T1708 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@cbb2b51 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@436b308c),segFN=segments_1,generation=1}
   [junit4]   2> 542010 T1708 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 542010 T1708 oass.SolrIndexSearcher.<init> Opening Searcher@4bae02fd[collection1] main
   [junit4]   2> 542010 T1708 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 542011 T1708 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 542011 T1708 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 542012 T1708 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 542012 T1708 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 542012 T1708 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 542013 T1708 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 542013 T1708 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 542013 T1708 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 542013 T1708 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 542015 T1709 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4bae02fd[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 542016 T1708 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 542017 T1712 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37136 collection:collection1 shard:shard1
   [junit4]   2> 542018 T1643 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 542018 T1643 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 542018 T1712 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 542020 T1643 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 542021 T1643 oasc.AbstractFullDistribZkTestBase.waitForThingsToLevelOut Wait for recoveries to finish - wait 30000 for each attempt
   [junit4]   2> 542021 T1643 oasc.AbstractDistribZkTestBase.waitForRecoveriesToFinish Wait for recoveries to finish - collection: collection1 failOnTimeout:true timeout (sec):30000
   [junit4]   2> 542022 T1643 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 542024 T1712 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 542026 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 542026 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 542026 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 542026 T1712 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 542027 T1712 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C540 name=collection1 org.apache.solr.core.SolrCore@55afd0dc url=http://127.0.0.1:37136/collection1 node=127.0.0.1:37136_ C540_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:37136, core=collection1, node_name=127.0.0.1:37136_}
   [junit4]   2> 542027 T1712 C540 P37136 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37136/collection1/
   [junit4]   2> 542027 T1666 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 542027 T1712 C540 P37136 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 542028 T1712 C540 P37136 oasc.SyncStrategy.syncToMe http://127.0.0.1:37136/collection1/ has no replicas
   [junit4]   2> 542028 T1712 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37136/collection1/ shard1
   [junit4]   2> 542028 T1712 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 542030 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 542033 T1665 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChild

[...truncated too long message...]

ctor.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1156415 T3678 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1156415 T3678 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1156415 T3678 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1156415 T3678 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1156433 T3575 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/sh/pe,null}
   [junit4]   2> 1156484 T3575 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1156486 T3575 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:59980 59980
   [junit4]   2> 1157200 T3576 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:59980 59980
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SyncSliceTest -Dtests.method=testDistribSearch -Dtests.seed=28A2A804290B2957 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=es_CO -Dtests.timezone=Europe/Prague -Dtests.file.encoding=UTF-8
   [junit4] ERROR   56.5s J0 | SyncSliceTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([28A2A804290B2957:A944261C5E54496B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:659)
   [junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1149)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
   [junit4]    > 	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1157206 T3575 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J0/./solr.cloud.SyncSliceTest-28A2A804290B2957-001
   [junit4]   2> NOTE: test params are: codec=Lucene3x, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=es_CO, timezone=Europe/Prague
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_05 (64-bit)/cpus=8,threads=1,free=76250896,total=316669952
   [junit4]   2> NOTE: All tests run in this JVM: [TestDynamicFieldCollectionResource, FileBasedSpellCheckerTest, AnalyticsMergeStrategyTest, SampleTest, ShowFileRequestHandlerTest, RankQueryTest, QueryElevationComponentTest, SimpleFacetsTest, SolrCoreTest, SpellPossibilityIteratorTest, AtomicUpdatesTest, HdfsUnloadDistributedZkTest, MultiThreadedOCPTest, TestFieldResource, AliasIntegrationTest, TestPseudoReturnFields, StatelessScriptUpdateProcessorFactoryTest, TestSearchPerf, CSVRequestHandlerTest, TestSolrQueryParserDefaultOperatorResource, ZkCLITest, TestSolrQueryParser, SSLMigrationTest, PolyFieldTest, TermVectorComponentTest, SchemaVersionSpecificBehaviorTest, MBeansHandlerTest, TestJoin, UUIDFieldTest, DistributedTermsComponentTest, UpdateRequestProcessorFactoryTest, TestCloudManagedSchemaCopyFields, MoreLikeThisHandlerTest, TestCloudManagedSchemaAddField, TestExceedMaxTermLength, TestDistribDocBasedVersion, DocValuesMissingTest, DateMathParserTest, ShardSplitTest, CircularListTest, DOMUtilTest, TestManagedResourceStorage, SliceStateTest, TestManagedStopFilterFactory, TestFieldSortValues, CacheHeaderTest, LeaderElectionTest, TestSort, AutoCommitTest, ReturnFieldsTest, TestDocBasedVersionConstraints, TestMergePolicyConfig, OverseerCollectionProcessorTest, ClusterStateTest, PrimitiveFieldTypeTest, WordBreakSolrSpellCheckerTest, ClusterStateUpdateTest, CopyFieldTest, SolrXmlInZkTest, EnumFieldTest, TestRemoteStreaming, TestSystemIdResolver, LegacyHTMLStripCharFilterTest, DocExpirationUpdateProcessorFactoryTest, CoreAdminCreateDiscoverTest, OverseerTest, TestRestManager, TestChildDocTransformer, URLClassifyProcessorTest, TestConfig, HdfsBasicDistributedZkTest, TestUniqueKeyFieldResource, TestSolrDeletionPolicy1, TestManagedResource, TestLMJelinekMercerSimilarityFactory, TestCloudManagedSchema, TestDocSet, BlockCacheTest, CoreMergeIndexesAdminHandlerTest, LukeRequestHandlerTest, TestDefaultSearchFieldResource, SearchHandlerTest, TestSolrXmlPersistor, SuggesterFSTTest, BadComponentTest, RecoveryZkTest, TestFreeTextSuggestions, HdfsBasicDistributedZk2Test, TestGroupingSearch, PeerSyncTest, TestLRUCache, TestCharFilters, SyncSliceTest]
   [junit4] Completed on J0 in 56.53s, 1 test, 1 error <<< FAILURES!

[...truncated 650 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921: There were test failures: 404 suites, 1648 tests, 2 errors, 1 failure, 37 ignored (16 assumptions)

Total time: 69 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_05) - Build # 10425 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10425/
Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC

2 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testDistribSearch

Error Message:
collection already exists: halfdeletedcollection2

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: collection already exists: halfdeletedcollection2
	at __randomizedtesting.SeedInfo.seed([7D1F4B1CF4D29ED6:FCF9C504838DFEEA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:554)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1513)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1539)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.deleteCollectionWithDownNodes(CollectionsAPIDistributedZkTest.java:376)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.doTest(CollectionsAPIDistributedZkTest.java:214)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([7D1F4B1CF4D29ED6:FCF9C504838DFEEA]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:659)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1149)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.GeneratedMethodAccessor51.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11456 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/init-core-data-001
   [junit4]   2> 795441 T2870 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 795441 T2870 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 795443 T2870 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 795444 T2870 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 795444 T2871 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 795544 T2870 oasc.ZkTestServer.run start zk server on port:44832
   [junit4]   2> 795545 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795549 T2877 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@64e4c29e name:ZooKeeperConnection Watcher:127.0.0.1:44832 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795556 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795556 T2870 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 795560 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795562 T2879 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39c383aa name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795562 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795562 T2870 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 795564 T2870 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 795565 T2870 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 795566 T2870 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 795567 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 795568 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 795570 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 795570 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 795572 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 795573 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 795576 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 795576 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 795577 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 795578 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 795579 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 795579 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 795581 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 795581 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 795582 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 795583 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 795584 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 795584 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 795586 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 795586 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 795588 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 795588 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> Using new style solr.xml
   [junit4]   2> 795590 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795592 T2881 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4516d624 name:ZooKeeperConnection Watcher:127.0.0.1:44832 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795592 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795592 T2870 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 795596 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795597 T2883 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3851f88 name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795597 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795598 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 795598 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 795602 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 795602 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
   [junit4]   2> 795606 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 795607 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 795608 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 795609 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 795610 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 795611 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
   [junit4]   2> 795612 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 795612 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
   [junit4]   2> 795613 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 795614 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 795615 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 795615 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 795617 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 795617 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 795618 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 795618 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 795620 T2870 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 795620 T2870 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
   [junit4]   2> 795850 T2870 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 795863 T2870 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:34131
   [junit4]   2> 795863 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 795864 T2870 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 795864 T2870 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002
   [junit4]   2> 795864 T2870 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/'
   [junit4]   2> 795874 T2870 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solr.xml
   [junit4]   2> 795880 T2870 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002
   [junit4]   2> 795881 T2870 oasc.CoreContainer.<init> New CoreContainer 1834357872
   [junit4]   2> 795881 T2870 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/]
   [junit4]   2> 795881 T2870 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 795881 T2870 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 795882 T2870 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 795882 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 795882 T2870 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 795882 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 795883 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 795883 T2870 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 795883 T2870 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 795883 T2870 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 795884 T2870 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 795884 T2870 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 795884 T2870 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44832/solr
   [junit4]   2> 795884 T2870 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 795885 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795887 T2894 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5d074e8d name:ZooKeeperConnection Watcher:127.0.0.1:44832 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795887 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795889 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 795890 T2896 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@659fbae2 name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 795891 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 795892 T2870 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 795895 T2870 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 795898 T2870 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 795900 T2870 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 795901 T2870 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 795903 T2870 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 795904 T2870 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34131_
   [junit4]   2> 795905 T2870 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34131_
   [junit4]   2> 795906 T2870 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 795907 T2870 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 795909 T2870 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34131_
   [junit4]   2> 795910 T2870 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 795911 T2870 oasc.Overseer.start Overseer (id=91921954063187973-127.0.0.1:34131_-n_0000000000) starting
   [junit4]   2> 795912 T2870 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 795922 T2898 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 795923 T2870 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 795925 T2870 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 795927 T2870 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 795929 T2897 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 795929 T2870 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002
   [junit4]   2> 795935 T2870 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/conf/
   [junit4]   2> 796058 T2870 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 796059 T2870 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 796059 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 796060 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 796062 T2901 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72e72c1b name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 796062 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 796063 T2870 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 796065 T2870 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 796292 T2870 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 796293 T2870 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 796295 T2870 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:47648
   [junit4]   2> 796295 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 796295 T2870 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 796296 T2870 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003
   [junit4]   2> 796296 T2870 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/'
   [junit4]   2> 796305 T2870 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solr.xml
   [junit4]   2> 796313 T2870 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003
   [junit4]   2> 796314 T2870 oasc.CoreContainer.<init> New CoreContainer 407851619
   [junit4]   2> 796314 T2870 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/]
   [junit4]   2> 796315 T2870 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 796315 T2870 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 796316 T2870 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 796316 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 796316 T2870 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 796317 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 796317 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 796317 T2870 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 796318 T2870 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 796318 T2870 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 796319 T2870 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 796319 T2870 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 796319 T2870 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44832/solr
   [junit4]   2> 796320 T2870 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 796321 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 796322 T2912 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ec1192c name:ZooKeeperConnection Watcher:127.0.0.1:44832 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 796323 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 796327 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 796329 T2914 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58dd2f6f name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 796329 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 796334 T2870 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 797336 T2870 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47648_
   [junit4]   2> 797337 T2870 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47648_
   [junit4]   2> 797339 T2896 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 797339 T2901 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 797339 T2914 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 797341 T2870 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003
   [junit4]   2> 797347 T2870 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/conf/
   [junit4]   2> 797463 T2870 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 797464 T2870 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 797464 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 797699 T2870 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 797700 T2870 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 797701 T2870 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41980
   [junit4]   2> 797702 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 797703 T2870 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 797703 T2870 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-004
   [junit4]   2> 797703 T2870 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-004/'
   [junit4]   2> 797713 T2870 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-004/solr.xml
   [junit4]   2> 797719 T2870 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-004
   [junit4]   2> 797720 T2870 oasc.CoreContainer.<init> New CoreContainer 1050660352
   [junit4]   2> 797720 T2870 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-004/]
   [junit4]   2> 797720 T2870 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 797720 T2870 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 797721 T2870 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 797721 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 797721 T2870 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 797721 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 797722 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 797722 T2870 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 797722 T2870 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 797722 T2870 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 797723 T2870 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 797723 T2870 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 797723 T2870 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44832/solr
   [junit4]   2> 797723 T2870 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 797724 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 797726 T2926 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27b45f36 name:ZooKeeperConnection Watcher:127.0.0.1:44832 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 797727 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 797728 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 797729 T2928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19e92170 name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 797730 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 797734 T2870 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 798736 T2870 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41980_
   [junit4]   2> 798737 T2870 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41980_
   [junit4]   2> 798739 T2896 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 798739 T2901 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 798739 T2928 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 798739 T2914 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 798741 T2870 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-004
   [junit4]   2> 798746 T2870 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-004/conf/
   [junit4]   2> 798841 T2870 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 798843 T2870 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 798843 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 799101 T2870 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 799102 T2870 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 799104 T2870 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:41022
   [junit4]   2> 799105 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 799105 T2870 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 799105 T2870 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005
   [junit4]   2> 799106 T2870 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/'
   [junit4]   2> 799115 T2870 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solr.xml
   [junit4]   2> 799122 T2870 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005
   [junit4]   2> 799122 T2870 oasc.CoreContainer.<init> New CoreContainer 1631700153
   [junit4]   2> 799122 T2870 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/]
   [junit4]   2> 799123 T2870 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 799123 T2870 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 799123 T2870 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 799123 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 799124 T2870 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 799124 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 799124 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 799124 T2870 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 799125 T2870 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 799125 T2870 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 799125 T2870 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 799126 T2870 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 799126 T2870 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44832/solr
   [junit4]   2> 799126 T2870 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 799127 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 799128 T2940 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3047d6eb name:ZooKeeperConnection Watcher:127.0.0.1:44832 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 799128 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 799130 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 799131 T2942 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@319e371a name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 799131 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 799135 T2870 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 800137 T2870 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41022_
   [junit4]   2> 800139 T2870 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41022_
   [junit4]   2> 800144 T2914 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 800144 T2942 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 800144 T2901 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 800144 T2896 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 800144 T2928 oascc.ZkStateReader$3.process Updating live nodes... (4)
   [junit4]   2> 800148 T2870 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005
   [junit4]   2> 800157 T2870 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/conf/
   [junit4]   2> 800281 T2870 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 800282 T2870 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 800283 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 800520 T2870 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 4
   [junit4]   2> 800520 T2870 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 800522 T2870 oejs.AbstractConnector.doStart Started SelectChannelConnector@127.0.0.1:43236
   [junit4]   2> 800523 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 800523 T2870 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 800523 T2870 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006
   [junit4]   2> 800524 T2870 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/'
   [junit4]   2> 800533 T2870 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solr.xml
   [junit4]   2> 800545 T2870 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006
   [junit4]   2> 800545 T2870 oasc.CoreContainer.<init> New CoreContainer 1097853236
   [junit4]   2> 800545 T2870 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/]
   [junit4]   2> 800546 T2870 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 800546 T2870 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 800546 T2870 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 800546 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 800546 T2870 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 800547 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 800547 T2870 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 800547 T2870 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 800547 T2870 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 800548 T2870 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 800548 T2870 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 800548 T2870 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 800549 T2870 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44832/solr
   [junit4]   2> 800549 T2870 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 800550 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 800551 T2954 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@176780cb name:ZooKeeperConnection Watcher:127.0.0.1:44832 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 800551 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 800554 T2870 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 800555 T2956 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@a5156ca name:ZooKeeperConnection Watcher:127.0.0.1:44832/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 800555 T2870 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 800560 T2870 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 801561 T2870 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43236_
   [junit4]   2> 801566 T2870 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:43236_
   [junit4]   2> 801568 T2914 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 801568 T2901 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 801568 T2942 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 801568 T2896 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 801568 T2956 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 801568 T2928 oascc.ZkStateReader$3.process Updating live nodes... (5)
   [junit4]   2> 801570 T2870 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006
   [junit4]   2> 801575 T2870 oasc.CorePropertiesLocator.discoverUnder Found core conf in /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/conf/
   [junit4]   2> 801705 T2870 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 801706 T2870 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 801707 T2870 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 801716 T2907 oasha.CollectionsHandler.handleCreateAction Creating Collection : replicationFactor=2&collection.configName=conf1&router.field=myOwnField&name=solrj_collection&router.name=compositeId&action=CREATE&numShards=2&wt=javabin&version=2
   [junit4]   2> 801716 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801717 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801717 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801717 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801717 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/collection-queue-work state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801718 T2898 oasc.OverseerCollectionProcessor.run Overseer Collection Processor: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 801719 T2958 oasc.OverseerCollectionProcessor.processMessage WARN OverseerCollectionProcessor.processMessage : createcollection , {
   [junit4]   2> 	  "operation":"createcollection",
   [junit4]   2> 	  "fromApi":"true",
   [junit4]   2> 	  "name":"solrj_collection",
   [junit4]   2> 	  "replicationFactor":"2",
   [junit4]   2> 	  "collection.configName":"conf1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "router.field":"myOwnField",
   [junit4]   2> 	  "router.name":"compositeId"}
   [junit4]   2> 801719 T2958 oasc.OverseerCollectionProcessor.createConfNode creating collections conf node /collections/solrj_collection 
   [junit4]   2> 801720 T2958 oascc.SolrZkClient.makePath makePath: /collections/solrj_collection
   [junit4]   2> 801724 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801725 T2897 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 801725 T2897 oasc.Overseer$ClusterStateUpdater.buildCollection building a new collection: solrj_collection
   [junit4]   2> 801725 T2897 oasc.Overseer$ClusterStateUpdater.createCollection Create collection solrj_collection with shards [shard1, shard2]
   [junit4]   2> 801727 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801727 T2956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801727 T2914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801727 T2896 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801727 T2928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801727 T2942 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801727 T2901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801824 T2958 oasc.OverseerCollectionProcessor.createCollection Creating SolrCores for new collection solrj_collection, shardNames [shard1, shard2] , replicationFactor : 2
   [junit4]   2> 801824 T2958 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica1 as part of slice shard1 of collection solrj_collection on 127.0.0.1:41022_
   [junit4]   2> 801825 T2958 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard1_replica2 as part of slice shard1 of collection solrj_collection on 127.0.0.1:34131_
   [junit4]   2> 801826 T2958 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica1 as part of slice shard2 of collection solrj_collection on 127.0.0.1:43236_
   [junit4]   2> 801827 T2937 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard1_replica1&action=CREATE&numShards=2&collection=solrj_collection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 801827 T2892 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard1_replica2&action=CREATE&numShards=2&collection=solrj_collection&shard=shard1&wt=javabin&version=2
   [junit4]   2> 801827 T2958 oasc.OverseerCollectionProcessor.createCollection Creating shard solrj_collection_shard2_replica2 as part of slice shard2 of collection solrj_collection on 127.0.0.1:47648_
   [junit4]   2> 801828 T2937 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica1 state=down collection=solrj_collection
   [junit4]   2> 801828 T2952 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard2_replica1&action=CREATE&numShards=2&collection=solrj_collection&shard=shard2&wt=javabin&version=2
   [junit4]   2> 801829 T2892 oasc.ZkController.publish publishing core=solrj_collection_shard1_replica2 state=down collection=solrj_collection
   [junit4]   2> 801829 T2937 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 801829 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801829 T2892 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 801830 T2908 oasha.CoreAdminHandler.handleCreateAction core create command qt=/admin/cores&collection.configName=conf1&name=solrj_collection_shard2_replica2&action=CREATE&numShards=2&collection=solrj_collection&shard=shard2&wt=javabin&version=2
   [junit4]   2> 801830 T2952 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica1 state=down collection=solrj_collection
   [junit4]   2> 801831 T2897 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 801831 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801831 T2952 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 801831 T2908 oasc.ZkController.publish publishing core=solrj_collection_shard2_replica2 state=down collection=solrj_collection
   [junit4]   2> 801831 T2897 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41022",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41022_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 801832 T2908 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 801833 T2897 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34131",
   [junit4]   2> 	  "core":"solrj_collection_shard1_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34131_",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 801836 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801837 T2897 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:43236",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:43236_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 801841 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801842 T2897 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47648",
   [junit4]   2> 	  "core":"solrj_collection_shard2_replica2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:47648_",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"solrj_collection",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 801844 T2896 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 801945 T2901 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801946 T2896 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801946 T2928 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801946 T2942 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801946 T2914 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 801945 T2956 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 5)
   [junit4]   2> 802829 T2937 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica1
   [junit4]   2> 802830 T2892 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard1_replica2
   [junit4]   2> 802831 T2892 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 802831 T2937 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 802831 T2892 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 802831 T2937 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 802831 T2892 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 802831 T2952 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica1
   [junit4]   2> 802832 T2937 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 802832 T2892 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 802833 T2952 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 802832 T2908 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for solrj_collection_shard2_replica2
   [junit4]   2> 802833 T2952 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 802833 T2937 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 802833 T2892 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solrj_collection_shard1_replica2/'
   [junit4]   2> 802834 T2937 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solrj_collection_shard1_replica1/'
   [junit4]   2> 802834 T2908 oasc.ZkController.createCollectionZkNode Check for collection zkNode:solrj_collection
   [junit4]   2> 802833 T2952 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 802835 T2908 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 802835 T2908 oascc.ZkStateReader.readConfigName Load collection config from:/collections/solrj_collection
   [junit4]   2> 802836 T2952 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 802836 T2908 oascc.ZkStateReader.readConfigName path=/collections/solrj_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 802836 T2908 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solrj_collection_shard2_replica2/'
   [junit4]   2> 802836 T2952 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solrj_collection_shard2_replica1/'
   [junit4]   2> 802854 T2892 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 802866 T2937 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 802868 T2908 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 802870 T2892 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 802872 T2892 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 802880 T2952 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 802884 T2892 oass.IndexSchema.readSchema [solrj_collection_shard1_replica2] Schema name=test
   [junit4]   2> 802884 T2937 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 802885 T2937 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 802891 T2952 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 802892 T2937 oass.IndexSchema.readSchema [solrj_collection_shard1_replica1] Schema name=test
   [junit4]   2> 802893 T2952 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 802904 T2908 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 802906 T2908 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 802906 T2952 oass.IndexSchema.readSchema [solrj_collection_shard2_replica1] Schema name=test
   [junit4]   2> 802919 T2908 oass.IndexSchema.readSchema [solrj_collection_shard2_replica2] Schema name=test
   [junit4]   2> 803073 T2952 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803073 T2952 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803074 T2952 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 803077 T2952 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 803079 T2952 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 803080 T2908 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803081 T2908 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803081 T2908 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 803083 T2892 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803083 T2892 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803084 T2892 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 803084 T2908 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 803086 T2952 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803086 T2908 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 803089 T2892 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 803088 T2952 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803087 T2937 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803092 T2937 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 803092 T2937 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 803092 T2892 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 803094 T2952 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803095 T2952 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803096 T2952 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803096 T2952 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803097 T2937 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 803098 T2952 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803099 T2952 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803099 T2937 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 803100 T2952 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica1' using configuration from collection solrj_collection
   [junit4]   2> 803101 T2952 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 803101 T2952 oasc.SolrCore.<init> [solrj_collection_shard2_replica1] Opening new SolrCore at ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solrj_collection_shard2_replica1/, dataDir=./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solrj_collection_shard2_replica1/data/
   [junit4]   2> 803102 T2952 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78fca702
   [junit4]   2> 803103 T2892 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803104 T2908 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803104 T2952 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solrj_collection_shard2_replica1/data
   [junit4]   2> 803104 T2952 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solrj_collection_shard2_replica1/data/index/
   [junit4]   2> 803105 T2952 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica1] Solr index directory './solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solrj_collection_shard2_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 803110 T2937 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803106 T2908 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803105 T2892 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803113 T2952 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-006/solrj_collection_shard2_replica1/data/index
   [junit4]   2> 803116 T2952 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 803115 T2908 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803115 T2892 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803115 T2937 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 803117 T2908 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803118 T2908 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803118 T2908 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803119 T2952 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@132e44b5 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@22e66751),segFN=segments_1,generation=1}
   [junit4]   2> 803119 T2952 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 803120 T2937 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803118 T2892 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803121 T2892 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803121 T2892 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803122 T2892 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803122 T2892 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803123 T2892 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica2' using configuration from collection solrj_collection
   [junit4]   2> 803122 T2937 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803123 T2937 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803123 T2892 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 803123 T2908 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803124 T2908 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803124 T2908 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard2_replica2' using configuration from collection solrj_collection
   [junit4]   2> 803125 T2908 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 803125 T2908 oasc.SolrCore.<init> [solrj_collection_shard2_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solrj_collection_shard2_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solrj_collection_shard2_replica2/data/
   [junit4]   2> 803126 T2908 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78fca702
   [junit4]   2> 803122 T2952 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 803127 T2908 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solrj_collection_shard2_replica2/data
   [junit4]   2> 803127 T2952 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 803124 T2892 oasc.SolrCore.<init> [solrj_collection_shard1_replica2] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solrj_collection_shard1_replica2/, dataDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solrj_collection_shard1_replica2/data/
   [junit4]   2> 803124 T2937 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 803128 T2892 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78fca702
   [junit4]   2> 803127 T2952 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 803127 T2908 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solrj_collection_shard2_replica2/data/index/
   [junit4]   2> 803129 T2892 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solrj_collection_shard1_replica2/data
   [junit4]   2> 803129 T2908 oasc.SolrCore.initIndex WARN [solrj_collection_shard2_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solrj_collection_shard2_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 803129 T2937 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 803130 T2937 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 803130 T2937 oasc.CoreContainer.create Creating SolrCore 'solrj_collection_shard1_replica1' using configuration from collection solrj_collection
   [junit4]   2> 803131 T2937 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 803131 T2937 oasc.SolrCore.<init> [solrj_collection_shard1_replica1] Opening new SolrCore at ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solrj_collection_shard1_replica1/, dataDir=./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solrj_collection_shard1_replica1/data/
   [junit4]   2> 803131 T2937 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@78fca702
   [junit4]   2> 803132 T2937 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solrj_collection_shard1_replica1/data
   [junit4]   2> 803128 T2952 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 803130 T2908 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-003/solrj_collection_shard2_replica2/data/index
   [junit4]   2> 803129 T2892 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solrj_collection_shard1_replica2/data/index/
   [junit4]   2> 803133 T2908 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 803133 T2937 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solrj_collection_shard1_replica1/data/index/
   [junit4]   2> 803134 T2937 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica1] Solr index directory './solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solrj_collection_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 803133 T2952 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 803135 T2937 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-005/solrj_collection_shard1_replica1/data/index
   [junit4]   2> 803134 T2908 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6a998767 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@238bf53b),segFN=segments_1,generation=1}
   [junit4]   2> 803135 T2908 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 803133 T2892 oasc.SolrCore.initIndex WARN [solrj_collection_shard1_replica2] Solr index directory '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solrj_collection_shard1_replica2/data/index' doesn't exist. Creating new index...
   [junit4]   2> 803135 T2937 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 803135 T2952 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 803137 T2937 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7e728789 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ffc89c7),segFN=segments_1,generation=1}
   [junit4]   2> 803137 T2937 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 803136 T2892 oasc.CachingDirectoryFactory.get return new directory for ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.CollectionsAPIDistributedZkTest-7D1F4B1CF4D29ED6-001/tempDir-002/solrj_collection_shard1_replica2/data/index
   [junit4]   2> 803137 T2952 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 803138 T2952 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 803139 T2952 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 803139 T2908 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 803139 T2892 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 803139 T2952 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 803139 T2908 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 803140 T2952 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 803141 T2952 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 803141 T2937 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 803141 T2892 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@63fd4a89 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@38bfcda9),segFN=segments_1,generation=1}
   [junit4]   2> 803142 T2892 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 803141 T2908 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 803142 T2937 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 803143 T2937 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 803141 T2952 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 803143 T2937 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 803142 T2908 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 803144 T2937 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 803144 T2952 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 803144 T2937 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   

[...truncated too long message...]

.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1210333 T4323 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1210333 T4323 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1210333 T4323 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1210333 T4323 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1210351 T4219 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1210402 T4219 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1210403 T4219 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:37141 37141
   [junit4]   2> 1211114 T4220 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:37141 37141
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SyncSliceTest -Dtests.method=testDistribSearch -Dtests.seed=7D1F4B1CF4D29ED6 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=el_GR -Dtests.timezone=Asia/Tbilisi -Dtests.file.encoding=UTF-8
   [junit4] ERROR   63.3s J1 | SyncSliceTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7D1F4B1CF4D29ED6:FCF9C504838DFEEA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:659)
   [junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1149)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
   [junit4]    > 	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1211120 T4219 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-7D1F4B1CF4D29ED6-001
   [junit4]   2> 63330 T4218 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene46, sim=DefaultSimilarity, locale=el_GR, timezone=Asia/Tbilisi
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_05 (64-bit)/cpus=8,threads=1,free=174614168,total=413376512
   [junit4]   2> NOTE: All tests run in this JVM: [DistanceFunctionTest, TimeZoneUtilsTest, NoCacheHeaderTest, TestComponentsName, TestDistributedSearch, TestHighlightDedupGrouping, TestAnalyzedSuggestions, SolrIndexConfigTest, PreAnalyzedUpdateProcessorTest, DistributedQueryComponentCustomSortTest, TestReloadAndDeleteDocs, TestNonDefinedSimilarityFactory, TestSolrQueryParserResource, TestPhraseSuggestions, SuggesterWFSTTest, TestStandardQParsers, TestManagedSchema, HdfsDirectoryTest, TriLevelCompositeIdRoutingTest, TestCollationFieldDocValues, CachingDirectoryFactoryTest, TestSolrCoreProperties, TestNonNRTOpen, TestStressLucene, AssignTest, PreAnalyzedFieldTest, HighlighterConfigTest, TestSchemaResource, XmlUpdateRequestHandlerTest, AnalyticsQueryTest, HdfsRecoveryZkTest, CurrencyFieldXmlFileTest, TestSolrJ, StatsComponentTest, TestIndexSearcher, RAMDirectoryFactoryTest, TestSolrDeletionPolicy2, AsyncMigrateRouteKeyTest, DateFieldTest, TestCollectionAPI, TestAnalyzeInfixSuggestions, DeleteInactiveReplicaTest, CurrencyFieldOpenExchangeTest, TestCloudInspectUtil, OpenCloseCoreStressTest, SignatureUpdateProcessorFactoryTest, TermVectorComponentDistributedTest, TestQuerySenderNoQuery, ReplicationFactorTest, TestFastWriter, UniqFieldsUpdateProcessorFactoryTest, JSONWriterTest, StressHdfsTest, TestNumberUtils, SolrCmdDistributorTest, TestManagedSchemaFieldResource, ZkSolrClientTest, TestMultiCoreConfBootstrap, TestStressRecovery, TestFoldingMultitermQuery, TestSchemaVersionResource, DirectSolrConnectionTest, AddSchemaFieldsUpdateProcessorFactoryTest, RemoteQueryErrorTest, TestDefaultSimilarityFactory, BasicZkTest, DisMaxRequestHandlerTest, TestComplexPhraseQParserPlugin, TestCopyFieldCollectionResource, ChangedSchemaMergeTest, LeaderElectionIntegrationTest, TestCloudManagedSchemaAddFields, SpellCheckCollatorTest, MinimalSchemaTest, TestCodecSupport, OverseerStatusTest, TestFiltering, HdfsWriteToMultipleCollectionsTest, TestSimpleQParserPlugin, PingRequestHandlerTest, TestCSVLoader, DirectSolrSpellCheckerTest, CollectionsAPIDistributedZkTest, PrimUtilsTest, TestBinaryField, CoreAdminRequestStatusTest, TestClassNameShortening, BJQParserTest, HdfsCollectionsAPIDistributedZkTest, TestCSVResponseWriter, TestDynamicFieldResource, PluginInfoTest, TestDistributedMissingSort, DistribCursorPagingTest, BasicDistributedZkTest, SyncSliceTest]
   [junit4] Completed on J1 in 63.60s, 1 test, 1 error <<< FAILURES!

[...truncated 614 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921: There were test failures: 404 suites, 1648 tests, 2 errors, 37 ignored (16 assumptions)

Total time: 71 minutes 9 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any



[JENKINS] Lucene-Solr-4.x-Linux (64bit/jdk1.8.0_05) - Build # 10424 - Still Failing!

Posted by Policeman Jenkins Server <je...@thetaphi.de>.
Build: http://jenkins.thetaphi.de/job/Lucene-Solr-4.x-Linux/10424/
Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseG1GC

1 tests failed.
REGRESSION:  org.apache.solr.cloud.SyncSliceTest.testDistribSearch

Error Message:
No live SolrServers available to handle this request

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
	at __randomizedtesting.SeedInfo.seed([90D6FB728BA33B00:1130756AFCFC5B3C]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:659)
	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1149)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:483)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.TestRuleFieldCacheSanity$1.evaluate(TestRuleFieldCacheSanity.java:51)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:793)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:453)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:43)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:360)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 11390 lines...]
   [junit4] Suite: org.apache.solr.cloud.SyncSliceTest
   [junit4]   2> Creating dataDir: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/init-core-data-001
   [junit4]   2> 1196668 T5079 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (true) and clientAuth (false)
   [junit4]   2> 1196668 T5079 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ax/d
   [junit4]   2> 1196671 T5079 oas.SolrTestCaseJ4.setUp ###Starting testDistribSearch
   [junit4]   2> 1196671 T5079 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1196672 T5080 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1196772 T5079 oasc.ZkTestServer.run start zk server on port:46413
   [junit4]   2> 1196773 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1215455 T5083 oazsp.FileTxnLog.commit WARN fsync-ing the write ahead log in SyncThread:0 took 18680ms which will adversely effect operation latency. See the ZooKeeper troubleshooting guide
   [junit4]   2> 1215458 T5081 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14691fc9b290000, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 1215459 T5086 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45a20173 name:ZooKeeperConnection Watcher:127.0.0.1:46413 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1215459 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1215460 T5079 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1215466 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1215468 T5088 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e061e94 name:ZooKeeperConnection Watcher:127.0.0.1:46413/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1215469 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1215469 T5079 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1215474 T5079 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1215477 T5079 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1215479 T5079 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1215480 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1215480 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1215483 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1215483 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1215486 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1215487 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1215489 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1215489 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1215490 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1215491 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1215492 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1215492 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1215494 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1215494 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1215496 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1215496 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1215498 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1215498 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1215500 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1215500 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1215502 T5079 oasc.AbstractZkTestCase.putConfig put /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1215502 T5079 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1215505 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1215506 T5090 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@62a3ec32 name:ZooKeeperConnection Watcher:127.0.0.1:46413/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1215507 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1215759 T5079 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1215761 T5079 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1215767 T5079 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:38014
   [junit4]   2> 1215769 T5079 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1215769 T5079 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1215769 T5079 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002
   [junit4]   2> 1215769 T5079 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/'
   [junit4]   2> 1215779 T5079 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/solr.xml
   [junit4]   2> 1215788 T5079 oasc.CoreContainer.<init> New CoreContainer 88287683
   [junit4]   2> 1215788 T5079 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/]
   [junit4]   2> 1215789 T5079 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1215789 T5079 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1215789 T5079 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1215789 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1215790 T5079 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1215790 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1215790 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1215790 T5079 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1215791 T5079 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1215792 T5079 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1215793 T5079 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1215793 T5079 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1215793 T5079 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46413/solr
   [junit4]   2> 1215793 T5079 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1215794 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1215797 T5101 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d64b888 name:ZooKeeperConnection Watcher:127.0.0.1:46413 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1215797 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1215800 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1215802 T5103 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@52660b47 name:ZooKeeperConnection Watcher:127.0.0.1:46413/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1215802 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1215804 T5079 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1215808 T5079 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1215812 T5079 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1215819 T5079 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1215824 T5079 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1215829 T5079 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1215832 T5079 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38014_ax%2Fd
   [junit4]   2> 1215833 T5079 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38014_ax%2Fd
   [junit4]   2> 1215837 T5079 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1215840 T5079 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1215845 T5079 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:38014_ax%2Fd
   [junit4]   2> 1215846 T5079 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1215848 T5079 oasc.Overseer.start Overseer (id=91921356528615429-127.0.0.1:38014_ax%2Fd-n_0000000000) starting
   [junit4]   2> 1215852 T5079 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1215864 T5105 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1215866 T5079 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1215869 T5079 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1215872 T5079 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1215876 T5104 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1215881 T5106 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1215882 T5106 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1215883 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1215883 T5106 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1215886 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1215887 T5104 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38014/ax/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38014_ax%2Fd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1215887 T5104 oasc.Overseer$ClusterStateUpdater.createCollection Create collection control_collection with shards [shard1]
   [junit4]   2> 1215888 T5104 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1215891 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1215892 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1216884 T5106 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1216884 T5106 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1216885 T5106 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1216885 T5106 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1216889 T5106 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1216889 T5106 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/collection1/'
   [junit4]   2> 1216890 T5106 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/collection1/lib/.svn/' to classloader
   [junit4]   2> 1216890 T5106 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/collection1/lib/classes/' to classloader
   [junit4]   2> 1216890 T5106 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/collection1/lib/README' to classloader
   [junit4]   2> 1216963 T5106 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 1216976 T5106 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1216977 T5106 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1216983 T5106 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1217104 T5106 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1217104 T5106 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1217105 T5106 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1217109 T5106 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1217110 T5106 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1217118 T5106 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1217119 T5106 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1217121 T5106 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1217122 T5106 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1217122 T5106 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1217122 T5106 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1217123 T5106 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1217123 T5106 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1217123 T5106 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1217123 T5106 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1217124 T5106 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-002/collection1/, dataDir=./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/control/data/
   [junit4]   2> 1217124 T5106 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d40c48c
   [junit4]   2> 1217124 T5106 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/control/data
   [junit4]   2> 1217124 T5106 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/control/data/index/
   [junit4]   2> 1217125 T5106 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1217125 T5106 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/control/data/index
   [junit4]   2> 1217125 T5106 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@41eb9670
   [junit4]   2> 1217126 T5106 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4b9a4abf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58bde340),segFN=segments_1,generation=1}
   [junit4]   2> 1217126 T5106 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1217139 T5106 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1217139 T5106 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1217139 T5106 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1217140 T5106 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1217140 T5106 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1217140 T5106 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1217140 T5106 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1217140 T5106 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1217141 T5106 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1217141 T5106 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1217141 T5106 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1217141 T5106 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1217141 T5106 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1217142 T5106 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1217142 T5106 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1217142 T5106 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1217144 T5106 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1217145 T5106 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1217145 T5106 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1217146 T5106 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=37.255859375, floorSegmentMB=1.982421875, forceMergeDeletesPctAllowed=11.60276313557855, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1217146 T5106 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4b9a4abf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58bde340),segFN=segments_1,generation=1}
   [junit4]   2> 1217146 T5106 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1217147 T5106 oass.SolrIndexSearcher.<init> Opening Searcher@b882267[collection1] main
   [junit4]   2> 1217147 T5106 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1217148 T5106 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1217148 T5106 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1217148 T5106 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1217148 T5106 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1217149 T5106 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1217149 T5106 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1217150 T5106 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1217150 T5106 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1217150 T5106 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1217152 T5107 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@b882267[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1217152 T5106 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1217153 T5110 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:38014/ax/d collection:control_collection shard:shard1
   [junit4]   2> 1217154 T5079 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1217154 T5079 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1217154 T5110 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1217157 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1217164 T5112 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@302ba71f name:ZooKeeperConnection Watcher:127.0.0.1:46413/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1217164 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1217166 T5079 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1217169 T5079 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1217174 T5110 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1217175 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217176 T5110 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1217176 T5110 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1217176 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C2871 name=collection1 org.apache.solr.core.SolrCore@52bda069 url=https://127.0.0.1:38014/ax/d/collection1 node=127.0.0.1:38014_ax%2Fd C2871_STATE=coll:control_collection core:collection1 props:{state=down, base_url=https://127.0.0.1:38014/ax/d, core=collection1, node_name=127.0.0.1:38014_ax%2Fd}
   [junit4]   2> 1217176 T5110 C2871 P38014 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:38014/ax/d/collection1/
   [junit4]   2> 1217177 T5110 C2871 P38014 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1217177 T5110 C2871 P38014 oasc.SyncStrategy.syncToMe https://127.0.0.1:38014/ax/d/collection1/ has no replicas
   [junit4]   2> 1217177 T5110 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:38014/ax/d/collection1/ shard1
   [junit4]   2> 1217177 T5110 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1217178 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217181 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217182 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217285 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1217285 T5112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1217332 T5110 oasc.ZkController.register We are https://127.0.0.1:38014/ax/d/collection1/ and leader is https://127.0.0.1:38014/ax/d/collection1/
   [junit4]   2> 1217332 T5110 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:38014/ax/d
   [junit4]   2> 1217332 T5110 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1217332 T5110 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1217332 T5110 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1217333 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217334 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217334 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217334 T5110 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1217335 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1217335 T5104 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:38014/ax/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38014_ax%2Fd",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1217336 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1217411 T5079 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 1217412 T5079 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1217414 T5079 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1217415 T5079 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:45198
   [junit4]   2> 1217417 T5079 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1217417 T5079 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1217417 T5079 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003
   [junit4]   2> 1217417 T5079 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/'
   [junit4]   2> 1217427 T5079 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/solr.xml
   [junit4]   2> 1217434 T5079 oasc.CoreContainer.<init> New CoreContainer 1723918840
   [junit4]   2> 1217435 T5079 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/]
   [junit4]   2> 1217435 T5079 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1217435 T5079 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1217436 T5079 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1217436 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1217436 T5079 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1217436 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1217437 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1217437 T5079 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1217437 T5079 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1217438 T5112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1217438 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1217439 T5079 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1217439 T5079 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1217440 T5079 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1217440 T5079 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46413/solr
   [junit4]   2> 1217440 T5079 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1217441 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1217442 T5123 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44c09871 name:ZooKeeperConnection Watcher:127.0.0.1:46413 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1217443 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1217448 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1217449 T5125 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@283c46d6 name:ZooKeeperConnection Watcher:127.0.0.1:46413/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1217449 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1217454 T5079 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1218456 T5079 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:45198_ax%2Fd
   [junit4]   2> 1218457 T5079 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:45198_ax%2Fd
   [junit4]   2> 1218459 T5112 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1218459 T5125 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1218459 T5103 oascc.ZkStateReader$3.process Updating live nodes... (2)
   [junit4]   2> 1218463 T5126 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1218463 T5126 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1218464 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1218464 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1218464 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1218464 T5126 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1218465 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1218465 T5104 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45198/ax/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45198_ax%2Fd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1218465 T5104 oasc.Overseer$ClusterStateUpdater.createCollection Create collection collection1 with shards [shard1]
   [junit4]   2> 1218465 T5104 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1218468 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1218571 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1218571 T5125 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1218571 T5112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1219464 T5126 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1219465 T5126 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1219466 T5126 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1219466 T5126 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1219468 T5126 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1219468 T5126 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: './solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/collection1/'
   [junit4]   2> 1219469 T5126 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/collection1/lib/.svn/' to classloader
   [junit4]   2> 1219470 T5126 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/collection1/lib/classes/' to classloader
   [junit4]   2> 1219470 T5126 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/collection1/lib/README' to classloader
   [junit4]   2> 1219502 T5126 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 1219518 T5126 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1219520 T5126 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1219524 T5126 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1219632 T5126 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1219632 T5126 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1219633 T5126 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1219636 T5126 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1219637 T5126 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1219650 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1219654 T5126 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1219657 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1219658 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1219659 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1219659 T5126 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1219661 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1219661 T5126 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1219661 T5126 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1219662 T5126 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1219662 T5126 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-003/collection1/, dataDir=./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty1/
   [junit4]   2> 1219662 T5126 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d40c48c
   [junit4]   2> 1219663 T5126 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty1
   [junit4]   2> 1219663 T5126 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty1/index/
   [junit4]   2> 1219664 T5126 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 1219664 T5126 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty1/index
   [junit4]   2> 1219665 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@47f34b6
   [junit4]   2> 1219665 T5126 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@29131faa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35a095dc),segFN=segments_1,generation=1}
   [junit4]   2> 1219666 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1219670 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1219670 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1219671 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1219671 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1219671 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1219671 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1219672 T5126 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1219672 T5126 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1219673 T5126 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1219673 T5126 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1219674 T5126 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1219674 T5126 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1219675 T5126 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1219675 T5126 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1219676 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1219676 T5126 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1219679 T5126 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1219681 T5126 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1219681 T5126 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1219682 T5126 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=37.255859375, floorSegmentMB=1.982421875, forceMergeDeletesPctAllowed=11.60276313557855, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1219683 T5126 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@29131faa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35a095dc),segFN=segments_1,generation=1}
   [junit4]   2> 1219683 T5126 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1219684 T5126 oass.SolrIndexSearcher.<init> Opening Searcher@59ec7115[collection1] main
   [junit4]   2> 1219684 T5126 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1219685 T5126 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1219686 T5126 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1219687 T5126 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1219687 T5126 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1219687 T5126 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1219688 T5126 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1219688 T5126 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1219689 T5126 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1219689 T5126 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1219692 T5127 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@59ec7115[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1219695 T5126 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1219696 T5130 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:45198/ax/d collection:collection1 shard:shard1
   [junit4]   2> 1219696 T5079 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1219697 T5079 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1219697 T5130 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1219704 T5130 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1219705 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219706 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219706 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219706 T5130 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1219707 T5130 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2872 name=collection1 org.apache.solr.core.SolrCore@fec4e8e url=https://127.0.0.1:45198/ax/d/collection1 node=127.0.0.1:45198_ax%2Fd C2872_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:45198/ax/d, core=collection1, node_name=127.0.0.1:45198_ax%2Fd}
   [junit4]   2> 1219707 T5130 C2872 P45198 oasc.SyncStrategy.sync Sync replicas to https://127.0.0.1:45198/ax/d/collection1/
   [junit4]   2> 1219708 T5130 C2872 P45198 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1219708 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1219708 T5130 C2872 P45198 oasc.SyncStrategy.syncToMe https://127.0.0.1:45198/ax/d/collection1/ has no replicas
   [junit4]   2> 1219709 T5130 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: https://127.0.0.1:45198/ax/d/collection1/ shard1
   [junit4]   2> 1219709 T5130 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1219711 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219714 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219718 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219820 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1219820 T5125 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1219820 T5112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1219865 T5130 oasc.ZkController.register We are https://127.0.0.1:45198/ax/d/collection1/ and leader is https://127.0.0.1:45198/ax/d/collection1/
   [junit4]   2> 1219865 T5130 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:45198/ax/d
   [junit4]   2> 1219865 T5130 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1219865 T5130 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1219865 T5130 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1219867 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219867 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219867 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219867 T5130 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1219868 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1219869 T5104 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:45198/ax/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:45198_ax%2Fd",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1219870 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1219936 T5079 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2
   [junit4]   2> 1219936 T5079 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1219940 T5079 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1219942 T5079 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:49317
   [junit4]   2> 1219945 T5079 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1219945 T5079 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1219946 T5079 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004
   [junit4]   2> 1219946 T5079 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/'
   [junit4]   2> 1219961 T5079 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/solr.xml
   [junit4]   2> 1219968 T5079 oasc.CoreContainer.<init> New CoreContainer 1210318148
   [junit4]   2> 1219969 T5079 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/]
   [junit4]   2> 1219969 T5079 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1219969 T5079 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1219970 T5079 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1219970 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1219970 T5079 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1219970 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1219971 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1219971 T5079 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1219971 T5079 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1219972 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1219972 T5112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1219972 T5125 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1219973 T5079 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1219973 T5079 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1219973 T5079 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1219974 T5079 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46413/solr
   [junit4]   2> 1219974 T5079 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1219975 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1219976 T5141 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a04310e name:ZooKeeperConnection Watcher:127.0.0.1:46413 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1219977 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1219979 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1219981 T5143 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6a4d9ba3 name:ZooKeeperConnection Watcher:127.0.0.1:46413/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1219981 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1220042 T5079 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1221046 T5079 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49317_ax%2Fd
   [junit4]   2> 1221048 T5079 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49317_ax%2Fd
   [junit4]   2> 1221051 T5112 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1221051 T5125 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1221051 T5103 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1221051 T5143 oascc.ZkStateReader$3.process Updating live nodes... (3)
   [junit4]   2> 1221060 T5144 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1221060 T5144 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1221061 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1221061 T5144 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1221061 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1221062 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1221063 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1221064 T5104 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:49317/ax/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49317_ax%2Fd",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":null}
   [junit4]   2> 1221064 T5104 oasc.Overseer$ClusterStateUpdater.updateState Collection already exists with numShards=1
   [junit4]   2> 1221064 T5104 oasc.Overseer$ClusterStateUpdater.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1221067 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1221170 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1221170 T5112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1221170 T5143 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1221170 T5125 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1222062 T5144 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1222062 T5144 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1222063 T5144 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1222064 T5144 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1222065 T5144 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1222065 T5144 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/collection1/'
   [junit4]   2> 1222066 T5144 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/collection1/lib/.svn/' to classloader
   [junit4]   2> 1222067 T5144 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/collection1/lib/classes/' to classloader
   [junit4]   2> 1222067 T5144 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/collection1/lib/README' to classloader
   [junit4]   2> 1222099 T5144 oasc.SolrConfig.<init> Using Lucene MatchVersion: LUCENE_4_9
   [junit4]   2> 1222118 T5144 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1222120 T5144 oass.IndexSchema.readSchema Reading Solr Schema from schema.xml
   [junit4]   2> 1222124 T5144 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1222244 T5144 oass.ByteField.init WARN ByteField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1222244 T5144 oass.ShortField.init WARN ShortField is deprecated and will be removed in 5.0. You should use TrieIntField instead.
   [junit4]   2> 1222245 T5144 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1222248 T5144 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1222249 T5144 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1222259 T5144 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1222260 T5144 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1222262 T5144 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1222263 T5144 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1222263 T5144 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1222263 T5144 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1222264 T5144 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1222264 T5144 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1222264 T5144 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1222265 T5144 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1222265 T5144 oasc.SolrCore.<init> [collection1] Opening new SolrCore at ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-004/collection1/, dataDir=./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty2/
   [junit4]   2> 1222265 T5144 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3d40c48c
   [junit4]   2> 1222266 T5144 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty2
   [junit4]   2> 1222266 T5144 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty2/index/
   [junit4]   2> 1222266 T5144 oasc.SolrCore.initIndex WARN [collection1] Solr index directory './solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty2/index' doesn't exist. Creating new index...
   [junit4]   2> 1222266 T5144 oasc.CachingDirectoryFactory.get return new directory for ./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-001/jetty2/index
   [junit4]   2> 1222266 T5144 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@4cdbddc4
   [junit4]   2> 1222267 T5144 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46823650 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1b3c52b9),segFN=segments_1,generation=1}
   [junit4]   2> 1222267 T5144 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1222269 T5144 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1222270 T5144 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1222270 T5144 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1222270 T5144 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1222270 T5144 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1222270 T5144 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1222271 T5144 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1222271 T5144 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1222271 T5144 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1222271 T5144 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 1222272 T5144 oasc.RequestHandlers.initHandlersFromConfig created /get: solr.RealTimeGetHandler
   [junit4]   2> 1222272 T5144 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.ReplicationHandler
   [junit4]   2> 1222272 T5144 oasc.RequestHandlers.initHandlersFromConfig created /replication: solr.ReplicationHandler
   [junit4]   2> 1222273 T5144 oasc.RequestHandlers.initHandlersFromConfig created /update: solr.UpdateRequestHandler
   [junit4]   2> 1222273 T5144 oasc.RequestHandlers.initHandlersFromConfig created /admin/: org.apache.solr.handler.admin.AdminHandlers
   [junit4]   2> 1222273 T5144 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 1222275 T5144 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1222276 T5144 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1222276 T5144 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1222276 T5144 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=37.255859375, floorSegmentMB=1.982421875, forceMergeDeletesPctAllowed=11.60276313557855, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1222277 T5144 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46823650 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1b3c52b9),segFN=segments_1,generation=1}
   [junit4]   2> 1222277 T5144 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1222277 T5144 oass.SolrIndexSearcher.<init> Opening Searcher@18baffaa[collection1] main
   [junit4]   2> 1222277 T5144 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1222278 T5144 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1222278 T5144 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1222279 T5144 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1222279 T5144 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1222279 T5144 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1222280 T5144 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1222280 T5144 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1222280 T5144 oasr.ManagedResource.notifyObserversDuringInit WARN No registered observers for /rest/managed
   [junit4]   2> 1222280 T5144 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1222282 T5145 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@18baffaa[collection1] main{StandardDirectoryReader(segments_1:1:nrt)}
   [junit4]   2> 1222282 T5144 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1222284 T5148 oasc.ZkController.register Register replica - core:collection1 address:https://127.0.0.1:49317/ax/d collection:collection1 shard:shard1
   [junit4]   2> 1222284 T5079 oass.SolrDispatchFilter.init user.dir=/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1
   [junit4]   2> 1222284 T5079 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1222286 T5148 oasc.ZkController.register We are https://127.0.0.1:49317/ax/d/collection1/ and leader is https://127.0.0.1:45198/ax/d/collection1/
   [junit4]   2> 1222286 T5148 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=https://127.0.0.1:49317/ax/d
   [junit4]   2> 1222286 T5148 oasc.ZkController.checkRecovery Core needs to recover:collection1
   [junit4]   2> 1222287 T5148 oasu.DefaultSolrCoreState.doRecovery Running recovery - first canceling any ongoing recovery
   [junit4]   2> ASYNC  NEW_CORE C2873 name=collection1 org.apache.solr.core.SolrCore@10ab9206 url=https://127.0.0.1:49317/ax/d/collection1 node=127.0.0.1:49317_ax%2Fd C2873_STATE=coll:collection1 core:collection1 props:{state=down, base_url=https://127.0.0.1:49317/ax/d, core=collection1, node_name=127.0.0.1:49317_ax%2Fd}
   [junit4]   2> 1222287 T5149 C2873 P49317 oasc.RecoveryStrategy.run Starting recovery process.  core=collection1 recoveringAfterStartup=true
   [junit4]   2> 1222287 T5148 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1222287 T5149 C2873 P49317 oasc.RecoveryStrategy.doRecovery ###### startupVersions=[]
   [junit4]   2> 1222287 T5149 C2873 P49317 oasc.RecoveryStrategy.doRecovery Publishing state of core collection1 as recovering, leader is https://127.0.0.1:45198/ax/d/collection1/ and I am https://127.0.0.1:49317/ax/d/collection1/
   [junit4]   2> 1222287 T5149 C2873 P49317 oasc.ZkController.publish publishing core=collection1 state=recovering collection=collection1
   [junit4]   2> 1222288 T5149 C2873 P49317 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1222288 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1222288 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1222289 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1222289 T5149 C2873 P49317 oasc.RecoveryStrategy.sendPrepRecoveryCmd Sending prep recovery command to https://127.0.0.1:45198/ax/d; WaitForState: action=PREPRECOVERY&core=collection1&nodeName=127.0.0.1%3A49317_ax%252Fd&coreNodeName=core_node2&state=recovering&checkLive=true&onlyIfLeader=true&onlyIfLeaderActive=true
   [junit4]   2> 1222289 T5104 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1222290 T5104 oasc.Overseer$ClusterStateUpdater.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"recovering",
   [junit4]   2> 	  "base_url":"https://127.0.0.1:49317/ax/d",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:49317_ax%2Fd",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node2"}
   [junit4]   2> 1222292 T5103 oasc.DistributedQueue$LatchChildWatcher.process LatchChildWatcher fired on path: /overseer/queue state: SyncConnected type NodeChildrenChanged
   [junit4]   2> 1222347 T5118 oasha.CoreAdminHandler.handleWaitForStateAction Going to wait for coreNodeName: core_node2, state: recovering, checkLive: true, onlyIfLeader: true, onlyIfLeaderActive: true
   [junit4]   2> 1222348 T5118 oascc.ZkStateReader.updateClusterState Updating cloud state from ZooKeeper... 
   [junit4]   2> 1222349 T5118 oasha.CoreAdminHandler.handleWaitForStateAction Will wait a max of 183 seconds to see collection1 (shard1 of collection1) have state: recovering
   [junit4]   2> 1222349 T5118 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=down, localState=active, nodeName=127.0.0.1:49317_ax%2Fd, coreNodeName=core_node2, onlyIfActiveCheckResult=false, nodeProps: core_node2:{"state":"down","base_url":"https://127.0.0.1:49317/ax/d","core":"collection1","node_name":"127.0.0.1:49317_ax%2Fd"}
   [junit4]   2> 1222396 T5125 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1222396 T5103 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1222396 T5112 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1222396 T5143 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1222516 T5079 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 3
   [junit4]   2> 1222517 T5079 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1222519 T5079 oejus.SslContextFactory.doStart Enabled Protocols [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2] of [SSLv2Hello, SSLv3, TLSv1, TLSv1.1, TLSv1.2]
   [junit4]   2> 1222520 T5079 oejs.AbstractConnector.doStart Started SslSelectChannelConnector@127.0.0.1:55488
   [junit4]   2> 1222522 T5079 oass.SolrDispatchFilter.init SolrDispatchFilter.init()
   [junit4]   2> 1222523 T5079 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1222523 T5079 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: ../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-005
   [junit4]   2> 1222524 T5079 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-005/'
   [junit4]   2> 1222540 T5079 oasc.ConfigSolr.fromFile Loading container configuration from /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-005/solr.xml
   [junit4]   2> 1222547 T5079 oasc.CoreContainer.<init> New CoreContainer 1968320948
   [junit4]   2> 1222548 T5079 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=../../../../../../../../../../mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001/tempDir-005/]
   [junit4]   2> 1222548 T5079 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1222549 T5079 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1222549 T5079 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1222549 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1222549 T5079 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1222549 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1222550 T5079 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1222550 T5079 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1222550 T5079 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1222552 T5079 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1222552 T5079 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1222552 T5079 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 1222552 T5079 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:46413/solr
   [junit4]   2> 1222553 T5079 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1222553 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1222556 T5161 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74daafed name:ZooKeeperConnection Watcher:127.0.0.1:46413 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1222556 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1222560 T5079 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1222562 T5163 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c7cf7d8 name:ZooKeeperConnection Watcher:127.0.0.1:46413/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1222563 T5079 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1222572 T5079 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1223349 T5118 oasha.CoreAdminHandler.handleWaitForStateAction In WaitForState(recovering): collection=collection1, shard=shard1, thisCore=collection1, leaderDoesNotNeedRecovery=false, isLeader? true, live=true, checkLive=true, currentState=recovering, localState=active, nodeName=127.0.0.1:49317_ax%2Fd, coreNodeName=core_no

[...truncated too long message...]

LeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.joinElection(LeaderElector.java:267)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.rejoinLeaderElection(ElectionContext.java:444)
   [junit4]   2> 	at org.apache.solr.cloud.ShardLeaderElectionContext.runLeaderProcess(ElectionContext.java:208)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.runIamLeaderProcess(LeaderElector.java:142)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:110)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1269369 T5183 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 4)
   [junit4]   2> 1269369 T5183 oascc.ZkStateReader$2.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1269369 T5183 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 1269369 T5183 oasc.LeaderElector$ElectionWatcher.process WARN  org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /overseer_elect/election
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getChildren(ZooKeeper.java:1472)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:260)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$6.execute(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:73)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:257)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.checkIfIamLeader(LeaderElector.java:94)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector.access$200(LeaderElector.java:55)
   [junit4]   2> 	at org.apache.solr.cloud.LeaderElector$ElectionWatcher.process(LeaderElector.java:303)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:522)
   [junit4]   2> 	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:498)
   [junit4]   2> 
   [junit4]   2> 1269382 T5079 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/ax/d,null}
   [junit4]   2> 1269435 T5079 oas.SolrTestCaseJ4.tearDown ###Ending testDistribSearch
   [junit4]   2> 1269436 T5079 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:46413 46413
   [junit4]   2> 1270252 T5080 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:46413 46413
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SyncSliceTest -Dtests.method=testDistribSearch -Dtests.seed=90D6FB728BA33B00 -Dtests.multiplier=3 -Dtests.slow=true -Dtests.locale=tr_TR -Dtests.timezone=Europe/Busingen -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   73.6s J1 | SyncSliceTest.testDistribSearch <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([90D6FB728BA33B00:1130756AFCFC5B3C]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrServer.request(LBHttpSolrServer.java:317)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrServer.request(CloudSolrServer.java:659)
   [junit4]    > 	at org.apache.solr.client.solrj.request.QueryRequest.process(QueryRequest.java:91)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrServer.query(SolrServer.java:301)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1149)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1118)
   [junit4]    > 	at org.apache.solr.cloud.SyncSliceTest.doTest(SyncSliceTest.java:236)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.testDistribSearch(BaseDistributedSearchTestCase.java:865)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1270278 T5079 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build/solr-core/test/J1/./solr.cloud.SyncSliceTest-90D6FB728BA33B00-001
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=161), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=161)), sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=tr_TR, timezone=Europe/Busingen
   [junit4]   2> NOTE: Linux 3.8.0-41-generic amd64/Oracle Corporation 1.8.0_05 (64-bit)/cpus=8,threads=1,free=98830296,total=284164096
   [junit4]   2> NOTE: All tests run in this JVM: [TestIndexingPerformance, SuggesterWFSTTest, HdfsDirectoryTest, ConvertedLegacyTest, TestShardHandlerFactory, CoreAdminRequestStatusTest, FileUtilsTest, OverseerStatusTest, XsltUpdateRequestHandlerTest, PingRequestHandlerTest, DeleteInactiveReplicaTest, TestRandomMergePolicy, ShardRoutingCustomTest, TestAnalyzedSuggestions, TestQuerySenderListener, TestSolrJ, TestSweetSpotSimilarityFactory, TestBinaryResponseWriter, TestUpdate, CollectionsAPIDistributedZkTest, TestRandomFaceting, TestXIncludeConfig, TestManagedSchemaFieldResource, TestUtils, BadIndexSchemaTest, HdfsLockFactoryTest, TestCollationFieldDocValues, RequiredFieldsTest, DocumentAnalysisRequestHandlerTest, TestComplexPhraseQParserPlugin, DeleteReplicaTest, CursorPagingTest, TestShortCircuitedRequests, TestZkChroot, DistributedDebugComponentTest, StandardRequestHandlerTest, ZkSolrClientTest, OpenCloseCoreStressTest, CollectionsAPIAsyncDistributedZkTest, DistributedExpandComponentTest, TermVectorComponentDistributedTest, OutputWriterTest, TestRangeQuery, BJQParserTest, HdfsSyncSliceTest, TestMultiCoreConfBootstrap, CurrencyFieldXmlFileTest, TestIBSimilarityFactory, DirectUpdateHandlerTest, AlternateDirectoryTest, TestFoldingMultitermQuery, ZkNodePropsTest, MinimalSchemaTest, TestReloadAndDeleteDocs, TestSchemaVersionResource, TestNonNRTOpen, BasicZkTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestDistributedGrouping, TestAtomicUpdateErrorCases, TestManagedSchema, SignatureUpdateProcessorFactoryTest, TestSolr4Spatial, DocValuesMultiTest, TestCollectionAPI, NotRequiredUniqueKeyTest, TestRTGBase, TestCSVResponseWriter, AssignTest, SimplePostToolTest, TestDefaultSimilarityFactory, CoreContainerCoreInitFailuresTest, TestInitQParser, TestSolrXmlPersistence, DistributedSuggestComponentTest, TestFiltering, ReplicationFactorTest, TestArbitraryIndexDir, MultiTermTest, HardAutoCommitTest, CustomCollectionTest, TestCoreDiscovery, SolrPluginUtilsTest, TestBinaryField, TimeZoneUtilsTest, DateFieldTest, RequestHandlersTest, PreAnalyzedFieldTest, TestExpandComponent, CachingDirectoryFactoryTest, ChangedSchemaMergeTest, HdfsCollectionsAPIDistributedZkTest, PluginInfoTest, SolrCoreCheckLockOnStartupTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, SyncSliceTest]
   [junit4] Completed on J1 in 73.64s, 1 test, 1 error <<< FAILURES!

[...truncated 613 lines...]
BUILD FAILED
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:467: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:447: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/build.xml:45: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/extra-targets.xml:37: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/build.xml:189: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/solr/common-build.xml:496: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:1297: The following error occurred while executing this line:
/mnt/ssd/jenkins/workspace/Lucene-Solr-4.x-Linux/lucene/common-build.xml:921: There were test failures: 404 suites, 1648 tests, 1 error, 37 ignored (16 assumptions)

Total time: 68 minutes 21 seconds
Build step 'Invoke Ant' marked build as failure
Description set: Java: 64bit/jdk1.8.0_05 -XX:+UseCompressedOops -XX:+UseG1GC
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any